From e98270579dcf56c09be96743b083bfc8f971f52e Mon Sep 17 00:00:00 2001 From: "Zhong, Ruijie" Date: Mon, 3 Nov 2025 18:16:16 -0800 Subject: [PATCH 1/4] [Nightly] Enable Inductor UT --- .github/actions/linux-uttest/action.yml | 43 +++++++++++++++++++++---- .github/scripts/check-ut.py | 31 ++++++++++++++++-- .github/scripts/ut_result_check.sh | 4 ++- .github/workflows/nightly_ondemand.yml | 2 +- 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/.github/actions/linux-uttest/action.yml b/.github/actions/linux-uttest/action.yml index 3bc1729bac..6a80419b28 100644 --- a/.github/actions/linux-uttest/action.yml +++ b/.github/actions/linux-uttest/action.yml @@ -116,6 +116,37 @@ runs: echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log cp *.xml ${{ github.workspace }}/ut_log + - name: xpu_inductor + shell: timeout 36000 bash -xe {0} + if: ${{ inputs.ut_name == 'xpu_inductor' }} + run: | + export PYTORCH_TEST_WITH_SLOW=1 + export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" + mkdir -p ut_log/xpu_inductor + cd pytorch + for file in "test/inductor"/*.py; do + filename=$(basename "$file") + echo "=== Starting test: $filename ===" + start=$(date +%s) + pytest -sv test/inductor/$filename --junit-xml=${{ github.workspace }}/ut_log/inductor_$filename.xml 2>${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_$filename_test_error.log | \ + tee ${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_$filename_test.log + end=$(date +%s) + echo -e "$filename duration: $((end - start))s" + echo "=== Finished test: $filename ===" + done + - name: test_xpu + shell: timeout 3600 bash -xe {0} + if: ${{ inputs.ut_name == 'test_xpu' }} + run: | + export PYTORCH_TEST_WITH_SLOW=1 + export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" + mkdir -p ut_log/test_xpu + cd pytorch + if [ -f "test/test_xpu.py" ]; then + pytest -sv test/test_xpu.py --junit-xml=${{ github.workspace }}/ut_log/test_xpu.xml \ + 2> ${{ github.workspace }}/ut_log/test_xpu/test_xpu_error.log | \ + tee ${{ github.workspace }}/ut_log/test_xpu/test_xpu.log + fi - name: torch_xpu shell: timeout 3600 bash -xe {0} if: ${{ inputs.ut_name == 'torch_xpu' }} @@ -124,12 +155,12 @@ runs: export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" mkdir -p ut_log/torch_xpu cd pytorch - test_cmd="python test/run_test.py --include " - for test in $(ls test/inductor | grep test); do test_cmd="${test_cmd} inductor/$test"; done - for test in $(ls test/xpu | grep test); do test_cmd="${test_cmd} xpu/$test"; done - if [ -f "test/test_xpu.py" ]; then test_cmd="${test_cmd} test_xpu.py"; fi - eval $test_cmd 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test_error.log | \ - tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_test.log + for file in "test/xpu"/*.py; do + filename=$(basename "$file") + pytest -sv test/xpu/$filename --junit-xml=${{ github.workspace }}/ut_log/torch_xpu_$filename.xml \ + 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_$filename_error.log | \ + tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_$filename.log + done - name: xpu_profiling shell: timeout 3600 bash -xe {0} if: ${{ inputs.ut_name == 'xpu_profiling' }} diff --git a/.github/scripts/check-ut.py b/.github/scripts/check-ut.py index a4027858ba..24cd4cb01b 100644 --- a/.github/scripts/check-ut.py +++ b/.github/scripts/check-ut.py @@ -107,6 +107,13 @@ def get_message(case): return " ; ".join(error_messages) if error_messages else f"{case.result[0].message.splitlines()[0]}" +def get_case_identifier(case): + """Generate a unique identifier for a test case to detect duplicates""" + category = get_category_from_case(case) + classname = get_classname(case) + name = get_name(case) + return f"{category}:{classname}:{name}" + def print_md_row(row, print_header=False, failure_list=None): if print_header: header = " | ".join([f"{key}" for key in row.keys()]) @@ -126,7 +133,15 @@ def print_failures(failure_list=None): print("### Test Failures") print_header = True + seen_cases = set() + unique_failures = [] for case in failures: + case_id = get_case_identifier(case) + if case_id not in seen_cases: + seen_cases.add(case_id) + unique_failures.append(case) + + for case in unique_failures: print_md_row({ 'Category': get_category_from_case(case), 'Class name': get_classname(case), @@ -141,9 +156,15 @@ def generate_failures_log(): if not failures: return + seen_cases = set() + failures_by_category.clear() + for case in failures: - category = get_category_from_case(case) - failures_by_category[category].append(case) + case_id = get_case_identifier(case) + if case_id not in seen_cases: + seen_cases.add(case_id) + category = get_category_from_case(case) + failures_by_category[category].append(case) for category, category_failures in failures_by_category.items(): if not category_failures: @@ -247,6 +268,10 @@ def determine_category(ut): return 'op_transformers' elif ut == 'test_xpu': return 'test_xpu' + elif ut == 'torch_xpu': + return 'torch_xpu' + elif 'inductor_' in ut: + return 'xpu_inductor' elif 'op_ut' in ut: return 'op_ut' else: @@ -343,6 +368,8 @@ def print_summary(): } for summary in summaries: + if summary['Test cases'] == 0: + continue print_md_row({ 'Category': summary['Category'], 'UT': summary['UT'], diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh index 7192402725..b68fe3e759 100644 --- a/.github/scripts/ut_result_check.sh +++ b/.github/scripts/ut_result_check.sh @@ -82,7 +82,9 @@ check_test_cases() { ["op_regression_dev1"]=1 ["op_transformers"]=237 ["op_ut"]=120408 + ["xpu_inductor"]=10330 ["test_xpu"]=69 + ["torch_xpu"]=359 ) if [[ ! -f "$log_file" ]]; then @@ -124,7 +126,7 @@ check_test_cases() { } -if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' || "${ut_suite}" == 'test_xpu' ]]; then +if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' || "${ut_suite}" == 'test_xpu' || "${ut_suite}" == 'xpu_inductor' || "${ut_suite}" == 'torch_xpu' ]]; then echo -e "=========================================================================" echo -e "Show Failed cases in ${ut_suite}" echo -e "=========================================================================" diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index a6d0ff1558..3f3246ba34 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -95,7 +95,7 @@ jobs: echo "No such scheduler: ${{ github.event.schedule }}" exit 1 fi - ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_distributed"]' + ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_inductor","torch_xpu","test_xpu","xpu_distributed"]' suite='["huggingface","timm_models","torchbench","pt2e"]' triton='' python='3.10' From 0978fcb7961c6b9199df2053b46b6da65ba383f9 Mon Sep 17 00:00:00 2001 From: "Zhong, Ruijie" Date: Mon, 3 Nov 2025 23:05:46 -0800 Subject: [PATCH 2/4] fix the filename show issue --- .github/actions/linux-uttest/action.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/linux-uttest/action.yml b/.github/actions/linux-uttest/action.yml index 6a80419b28..8e01fa6ab1 100644 --- a/.github/actions/linux-uttest/action.yml +++ b/.github/actions/linux-uttest/action.yml @@ -124,12 +124,12 @@ runs: export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" mkdir -p ut_log/xpu_inductor cd pytorch - for file in "test/inductor"/*.py; do + for file in "test/inductor"/test*.py; do filename=$(basename "$file") echo "=== Starting test: $filename ===" start=$(date +%s) - pytest -sv test/inductor/$filename --junit-xml=${{ github.workspace }}/ut_log/inductor_$filename.xml 2>${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_$filename_test_error.log | \ - tee ${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_$filename_test.log + pytest -sv test/inductor/$filename --junit-xml=${{ github.workspace }}/ut_log/inductor_$filename.xml 2>${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_${filename}_test_error.log | \ + tee ${{ github.workspace }}/ut_log/xpu_inductor/xpu_inductor_${filename}_test.log end=$(date +%s) echo -e "$filename duration: $((end - start))s" echo "=== Finished test: $filename ===" @@ -155,11 +155,11 @@ runs: export PYTORCH_TESTING_DEVICE_ONLY_FOR="xpu" mkdir -p ut_log/torch_xpu cd pytorch - for file in "test/xpu"/*.py; do + for file in "test/xpu"/test*.py; do filename=$(basename "$file") pytest -sv test/xpu/$filename --junit-xml=${{ github.workspace }}/ut_log/torch_xpu_$filename.xml \ - 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_$filename_error.log | \ - tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_$filename.log + 2> ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_${filename}_error.log | \ + tee ${{ github.workspace }}/ut_log/torch_xpu/torch_xpu_${filename}.log done - name: xpu_profiling shell: timeout 3600 bash -xe {0} From 385ce34d3b6ff4306a0f7174f21fed34ab567f2c Mon Sep 17 00:00:00 2001 From: "Zhong, Ruijie" Date: Tue, 4 Nov 2025 02:34:27 -0800 Subject: [PATCH 3/4] fix results summary issue --- .github/scripts/check-ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check-ut.py b/.github/scripts/check-ut.py index 24cd4cb01b..f51f930c65 100644 --- a/.github/scripts/check-ut.py +++ b/.github/scripts/check-ut.py @@ -268,7 +268,7 @@ def determine_category(ut): return 'op_transformers' elif ut == 'test_xpu': return 'test_xpu' - elif ut == 'torch_xpu': + elif 'torch_xpu_' in ut: return 'torch_xpu' elif 'inductor_' in ut: return 'xpu_inductor' From f20187f5c697a16efabe1de0574d33b16e11e648 Mon Sep 17 00:00:00 2001 From: "Zhong, Ruijie" Date: Wed, 5 Nov 2025 17:29:40 -0800 Subject: [PATCH 4/4] update new expected number --- .github/scripts/ut_result_check.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh index b68fe3e759..59ad743f1b 100644 --- a/.github/scripts/ut_result_check.sh +++ b/.github/scripts/ut_result_check.sh @@ -82,9 +82,9 @@ check_test_cases() { ["op_regression_dev1"]=1 ["op_transformers"]=237 ["op_ut"]=120408 - ["xpu_inductor"]=10330 - ["test_xpu"]=69 - ["torch_xpu"]=359 + ["xpu_inductor"]=20880 + ["test_xpu"]=73 + ["torch_xpu"]=396 ) if [[ ! -f "$log_file" ]]; then