Switch to pytest-skip instead of pytest-select (#3861)

anmyachev · pbchekin · web-flow · commit 8a9787f464f6 · 2025-04-07T20:56:35.000+02:00
Signed-off-by: Anatoly Myachev &lt;anatoly.myachev@intel.com&gt;
Co-authored-by: Pavel Chekin &lt;pavel.chekin@intel.com&gt;
diff --git a/.github/workflows/auto-update-translator-cid.yml b/.github/workflows/auto-update-translator-cid.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Install test dependencies
         if: ${{ env.TARGET_PRID == null }}
         run: |
-          pip install pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout expecttest
+          pip install pytest pytest-xdist pytest-rerunfailures pytest-skip pytest-timeout expecttest
           pip install git+https://github.com/kwasd/pytest-capturewarnings-ng.git@v1.2.0
 
       - name: Get commit ID from Triton's spirv-llvm-translator.conf
diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
@@ -227,9 +227,9 @@ jobs:
       - name: Pass rate
         run: |
           source ./scripts/capture-hw-details.sh
-          python scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}
-          python scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json
-          python scripts/pass_rate.py --reports reports --suite tutorials --json ${{ env.SKIPLIST }} > pass_rate_tutorials.json
+          python scripts/pass_rate.py --reports reports
+          python scripts/pass_rate.py --reports reports --json > pass_rate.json
+          python scripts/pass_rate.py --reports reports --suite tutorials --json > pass_rate_tutorials.json
 
       - name: Upload pass rate report
         # upload reports only for the default branch
diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml
@@ -152,9 +152,9 @@ jobs:
           pip install defusedxml
           bash -c "\
             source ./scripts/capture-hw-details.sh; \
-            python scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}; \
-            python scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json; \
-            python scripts/pass_rate.py --reports reports --suite tutorials --json ${{ env.SKIPLIST }} > pass_rate_tutorials.json; \
+            python scripts/pass_rate.py --reports reports; \
+            python scripts/pass_rate.py --reports reports --json > pass_rate.json; \
+            python scripts/pass_rate.py --reports reports --suite tutorials --json > pass_rate_tutorials.json; \
           "
 
       - name: Upload pass rate report
diff --git a/.github/workflows/pip-test-windows.yml b/.github/workflows/pip-test-windows.yml
@@ -159,9 +159,9 @@ jobs:
           cd ${{ env.NEW_WORKSPACE }}
           bash -c "\
             source ./scripts/capture-hw-details.sh; \
-            python scripts/pass_rate.py --reports reports ${{ env.SKIPLIST }}; \
-            python scripts/pass_rate.py --reports reports --json ${{ env.SKIPLIST }} > pass_rate.json; \
-            python scripts/pass_rate.py --reports reports --suite tutorials --json ${{ env.SKIPLIST }} > pass_rate_tutorials.json; \
+            python scripts/pass_rate.py --reports reports; \
+            python scripts/pass_rate.py --reports reports --json > pass_rate.json; \
+            python scripts/pass_rate.py --reports reports --suite tutorials --json > pass_rate_tutorials.json; \
           "
 
       - name: Upload pass rate report
diff --git a/scripts/docs-triton.sh b/scripts/docs-triton.sh
@@ -36,7 +36,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton
 export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build
 export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd)
 
-python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select
+python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-skip
 
 source $SCRIPTS_DIR/pytest-utils.sh
 $SCRIPTS_DIR/install-pytorch.sh $([ $VENV = true ] && echo "--venv")
diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py
@@ -8,7 +8,7 @@
 import pathlib
 import platform
 import sys
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 from defusedxml.ElementTree import parse
 
@@ -66,30 +66,9 @@ def create_argument_parser() -> argparse.ArgumentParser:
         default='all',
         help='name of the test suite, default: %(default)s',
     )
-    argument_parser.add_argument(
-        '--skip-list',
-        type=str,
-        help='an exclude list dir used in pass rate calculation',
-    )
     return argument_parser
 
 
-def get_deselected(report_path: pathlib.Path, skiplist_dir: pathlib.Path) -> int:
-    """Calculates deselected (via skiplist) tests."""
-    skiplist_path = skiplist_dir / f'{report_path.stem}.txt'
-    if not skiplist_path.exists():
-        return 0
-    with skiplist_path.open('r') as f:
-        count = 0
-        for line in f.readlines():
-            # `strip` allows to skip lines with only '\n' character
-            line = line.strip()
-            # skip empty lines and comments
-            if line and not line.startswith('#'):
-                count += 1
-        return count
-
-
 def get_warnings(reports_path: pathlib.Path, suite: str) -> List[TestWarning]:
     """Returns a list of warnings for the specified suite."""
     path = reports_path / f'{suite}-warnings.json'
@@ -101,10 +80,10 @@ def get_warnings(reports_path: pathlib.Path, suite: str) -> List[TestWarning]:
 
 
 def get_missing_tests(warnings: List[TestWarning]) -> List[str]:
-    """Searches warnings for PytestSelectWarning and returns a list of missing tests."""
+    """Searches warnings for UserWarning and returns a list of missing tests."""
     tests = set()
     for warning in warnings:
-        if 'PytestSelectWarning: pytest-select: Not all deselected' not in warning.message:
+        if 'UserWarning: pytest-skip: Not all deselected' not in warning.message:
             continue
         for line in warning.message.splitlines():
             if line.startswith('  - '):
@@ -124,7 +103,7 @@ def get_all_missing_tests(reports_path: pathlib.Path) -> Dict[str, List[str]]:
     return all_missing_tests
 
 
-def parse_report(report_path: pathlib.Path, skiplist_dir: Optional[pathlib.Path]) -> ReportStats:
+def parse_report(report_path: pathlib.Path) -> ReportStats:
     """Parses the specified report."""
     stats = ReportStats(name=report_path.stem)
     root = parse(report_path).getroot()
@@ -145,13 +124,6 @@ def parse_report(report_path: pathlib.Path, skiplist_dir: Optional[pathlib.Path]
                 testsuite_fixme_tests.add(warning.location)
         stats.fixme += len(testsuite_fixme_tests)
 
-    test_unskip = os.getenv('TEST_UNSKIP', 'false')
-    if test_unskip not in ('true', 'false'):
-        raise ValueError('Error: please set TEST_UNSKIP true or false')
-    if skiplist_dir and test_unskip == 'false':
-        deselected = get_deselected(report_path, skiplist_dir)
-        stats.skipped += deselected
-        stats.total += deselected
     stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed
     return stats
 
@@ -180,7 +152,7 @@ def find_stats(stats: List[ReportStats], name: str) -> ReportStats:
 def parse_junit_reports(args: argparse.Namespace) -> List[ReportStats]:
     """Parses junit report in the specified directory."""
     reports_path = pathlib.Path(args.reports)
-    return [parse_report(report, args.skiplist_dir) for report in reports_path.glob('*.xml')]
+    return [parse_report(report) for report in reports_path.glob('*.xml')]
 
 
 def parse_tutorials_reports(args: argparse.Namespace) -> List[ReportStats]:
@@ -248,10 +220,6 @@ def main():
     """Main."""
     args = create_argument_parser().parse_args()
     args.report_path = pathlib.Path(args.reports)
-    if args.skip_list:
-        args.skiplist_dir = pathlib.Path(args.skip_list)
-    else:
-        args.skiplist_dir = None
 
     missing_tests = get_all_missing_tests(args.report_path)
     if missing_tests:
diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh
@@ -39,7 +39,7 @@ pytest() {
         sed -e '/^#/d' -e '/^\s*$/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
         if [[ $TEST_UNSKIP = false ]]; then
             pytest_extra_args+=(
-                "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
+                "--skip-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
             )
         else
             pytest_extra_args+=(
diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt
@@ -14,5 +14,5 @@ tabulate
 pytest-xdist
 pytest-forked
 pytest-rerunfailures
-pytest-select
+pytest-skip
 pytest-timeout
diff --git a/scripts/skiplist/a770/language.txt b/scripts/skiplist/a770/language.txt
diff --git a/scripts/skiplist/lts/language.txt b/scripts/skiplist/lts/language.txt
diff --git a/scripts/skiplist/mtl/language.txt b/scripts/skiplist/mtl/language.txt
diff --git a/scripts/test_pass_rate.py b/scripts/test_pass_rate.py

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ pytest() {`
`39`	`39`	`sed -e '/^#/d' -e '/^\s*$/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"`
`40`	`40`	`if [[ $TEST_UNSKIP = false ]]; then`
`41`	`41`	`pytest_extra_args+=(`
`42`		`- "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"`
	`42`	`+ "--skip-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"`
`43`	`43`	`)`
`44`	`44`	`else`
`45`	`45`	`pytest_extra_args+=(`