From 50a9f78f12b6f52f3239fc18585f997d235525b2 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 15:16:50 +0530 Subject: [PATCH 01/12] Changes to support new submission directory structure --- ...mlperf-inference-submission-generation.yml | 39 ++------------- .../customize.py | 49 ++++++------------- 2 files changed, 19 insertions(+), 69 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index a0cb0daec..d39d37588 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -29,42 +29,13 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open", "closed-open"] category: ["datacenter", "edge"] - round: ["5.0", "5.1"] + round: ["6.0"] folder_name: ["closed", "open"] action: ["run", "docker"] exclude: - os: macos-latest - os: windows-latest - category: "edge" - include: - - os: ubuntu-latest - python-version: "3.12" - division: "closed" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "run" - - os: ubuntu-latest - python-version: "3.12" - division: "closed" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "docker" - - os: ubuntu-latest - python-version: "3.12" - division: "open" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "run" - - os: ubuntu-latest - python-version: "3.12" - division: "open" - category: "edge" - round: "5.0" - folder_name: "closed-edge" - action: "docker" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -87,13 +58,13 @@ jobs: fi - name: Pull repo where test cases are uploaded run: | - git clone -b submission-generation-examples https://github.com/mlcommons/inference.git submission_generation_examples + git clone -b submission-generation-examples https://github.com/anandhu-eng/inference.git new-sub-dir - name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.folder_name }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | if [ "${{ matrix.folder_name }}" == "closed" ]; then description="Test submission - contains closed edge and datacenter" - elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then - description="Test submission - contains closed-power edge and datacenter results" + # elif [ "${{ matrix.folder_name }}" == "closed-power" ]; then + # description="Test submission - contains closed-power edge and datacenter results" fi # Dynamically set the log group to simulate a dynamic step name echo "::group::$description" @@ -107,7 +78,7 @@ jobs: mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 8ca22b596..c53bcf853 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -309,16 +309,11 @@ def generate_submission(env, state, inp, submission_division, logger): sub_res = new_res submission_path = os.path.join(path_submission, "results", sub_res) - measurement_path = os.path.join( - path_submission, "measurements", sub_res) - compliance_path = os.path.join(path_submission, "compliance", sub_res) system_path = os.path.join(path_submission, "systems") submission_system_path = system_path if not os.path.isdir(submission_path): os.makedirs(submission_path) - if not os.path.isdir(measurement_path): - os.makedirs(measurement_path) if not os.path.isdir(submission_system_path): os.makedirs(submission_system_path) system_file = os.path.join(submission_system_path, sub_res + ".json") @@ -342,8 +337,6 @@ def generate_submission(env, state, inp, submission_division, logger): platform_info_file = None result_model_path = os.path.join(result_path, model) submission_model_path = os.path.join(submission_path, model) - measurement_model_path = os.path.join(measurement_path, model) - compliance_model_path = os.path.join(compliance_path, model) code_model_path = os.path.join(code_path, model) scenarios = [ f for f in os.listdir(result_model_path) if not os.path.isfile( @@ -367,10 +360,6 @@ def generate_submission(env, state, inp, submission_division, logger): result_model_path, scenario) submission_scenario_path = os.path.join( submission_model_path, scenario) - measurement_scenario_path = os.path.join( - measurement_model_path, scenario) - compliance_scenario_path = os.path.join( - compliance_model_path, scenario) ''' if duplicate and scenario == 'singlestream': @@ -445,20 +434,11 @@ def generate_submission(env, state, inp, submission_division, logger): ) continue - if not os.path.isdir(measurement_scenario_path): - os.makedirs(measurement_scenario_path) - for mode in modes: result_mode_path = os.path.join(result_scenario_path, mode) submission_mode_path = os.path.join( submission_scenario_path, mode) - submission_measurement_path = measurement_scenario_path - submission_compliance_path = os.path.join( - compliance_scenario_path, mode) - if mode.startswith("TEST"): - submission_results_path = submission_compliance_path - else: - submission_results_path = submission_mode_path + submission_results_path = submission_mode_path if os.path.exists(submission_results_path): shutil.rmtree(submission_results_path) @@ -492,10 +472,10 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( analyzer_settings_file, os.path.join( - submission_measurement_path, "analyzer_table.md")) + submission_scenario_path, "analyzer_table.md")) shutil.copy( power_settings_file, os.path.join( - submission_measurement_path, "power_settings.md")) + submission_scenario_path, "power_settings.md")) result_ranging_path = os.path.join( result_mode_path, 'ranging') @@ -563,14 +543,14 @@ def generate_submission(env, state, inp, submission_division, logger): if os.path.exists(user_conf_path): shutil.copy( user_conf_path, os.path.join( - measurement_scenario_path, 'user.conf')) + submission_scenario_path, 'user.conf')) else: user_conf_path = os.path.join( result_mode_path, "user.conf") if os.path.exists(user_conf_path): shutil.copy( user_conf_path, os.path.join( - submission_measurement_path, 'user.conf')) + submission_scenario_path, 'user.conf')) else: if mode.lower() == "performance": return { @@ -579,7 +559,7 @@ def generate_submission(env, state, inp, submission_division, logger): # First check for measurements directory in scenario folder measurements_json_path = os.path.join( result_scenario_path, "measurements.json") - target_measurement_json_path = measurement_scenario_path + target_measurement_json_path = submission_scenario_path if not os.path.exists(measurements_json_path): measurements_json_path = os.path.join( @@ -668,7 +648,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( os.path.join( result_mode_path, f), os.path.join( - submission_measurement_path, f)) + submission_scenario_path, f)) if f == "system_info.txt" and not platform_info_file: # the first found system_info.txt will be taken as platform info file for a specific model to be placed in # measurements-model folder when generating @@ -679,7 +659,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( os.path.join( result_mode_path, f), os.path.join( - submission_measurement_path, mode + "_" + f)) + submission_scenario_path, mode + "_" + f)) if mode == "accuracy": if os.path.exists(os.path.join( @@ -705,13 +685,12 @@ def generate_submission(env, state, inp, submission_division, logger): result_scenario_path, "system_info.txt")): shutil.copy( os.path.join( - result_scenario_path, "system_info.txt"), os.path.join( - submission_measurement_path, f)) + result_scenario_path, "system_info.txt"), submission_scenario_path) platform_info_file = os.path.join( result_scenario_path, "system_info.txt") readme_file = os.path.join( - submission_measurement_path, "README.md") + submission_scenario_path, "README.md") if not os.path.exists(readme_file): with open(readme_file, mode='w') as f: f.write("TBD") # create an empty README @@ -741,7 +720,7 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( sys_info_file, os.path.join( - measurement_model_path, + submission_model_path, "system_info.txt")) # Copy system_info.txt to the submission measurements folder if any @@ -757,14 +736,14 @@ def generate_submission(env, state, inp, submission_division, logger): shutil.copy( sys_info_file, os.path.join( - measurement_path, + submission_path, "system_info.txt")) else: if env.get('MLC_GET_PLATFORM_DETAILS', '') == "yes": mlc_input = {'action': 'run', 'automation': 'script', 'tags': 'get,platform,details', - 'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(measurement_path, "system_info.txt")}, + 'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(submission_path, "system_info.txt")}, 'quiet': True } mlc = i['automation'].action_object @@ -779,7 +758,7 @@ def generate_submission(env, state, inp, submission_division, logger): logger.info(tabulate(result_table, headers=headers, tablefmt="pretty")) - sut_readme_file = os.path.join(measurement_path, "README.md") + sut_readme_file = os.path.join(submission_path, "README.md") with open(sut_readme_file, mode='w') as f: f.write(tabulate(result_table, headers=headers, tablefmt="github")) From 98ec5f1ba3138583543fe2690dce2130c2f5cdae Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 15:19:30 +0530 Subject: [PATCH 02/12] fix typo --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index d39d37588..8b4fcbda5 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -58,7 +58,7 @@ jobs: fi - name: Pull repo where test cases are uploaded run: | - git clone -b submission-generation-examples https://github.com/anandhu-eng/inference.git new-sub-dir + git clone -b new-sub-dir https://github.com/anandhu-eng/inference.git submission-generation-examples - name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.folder_name }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | if [ "${{ matrix.folder_name }}" == "closed" ]; then From e9fe81e4388e9bb1ea39c250802102bfe1b0c116 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 15:31:02 +0530 Subject: [PATCH 03/12] fix typo --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 8b4fcbda5..926c4174d 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -58,7 +58,7 @@ jobs: fi - name: Pull repo where test cases are uploaded run: | - git clone -b new-sub-dir https://github.com/anandhu-eng/inference.git submission-generation-examples + git clone -b new-sub-dir https://github.com/anandhu-eng/inference.git submission_generation_examples - name: Run Submission Generation - round-${{ matrix.round }}${{ matrix.folder_name }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} run: | if [ "${{ matrix.folder_name }}" == "closed" ]; then From a21a5c0083dbfef43182d7790574f795caa6d2bb Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 15:41:28 +0530 Subject: [PATCH 04/12] test commit --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 926c4174d..e9491ee2c 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -78,7 +78,7 @@ jobs: mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.master $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi From 81afb25d3bc00eed7ce24aca63b94008836c7c77 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 16:51:02 +0530 Subject: [PATCH 05/12] test commit --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- script/get-mlperf-inference-utils/mlperf_utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index e9491ee2c..926c4174d 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -78,7 +78,7 @@ jobs: mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.master $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index ffb0d3d7a..419d5f6de 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -188,6 +188,7 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, result['performance'] = round(performance_result_, 3) if scenario != effective_scenario: + print(f"{fname} : {scenario} : {effective_scenario}") inferred, inferred_result = checker.get_inferred_result( scenario, effective_scenario, performance_result, mlperf_log, config, False) From 12d0751a964af014de5ab3d97755d49a133c3d15 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 16:55:34 +0530 Subject: [PATCH 06/12] fix for args --- script/get-mlperf-inference-utils/mlperf_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index 419d5f6de..d35974301 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -189,7 +189,7 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res, if scenario != effective_scenario: print(f"{fname} : {scenario} : {effective_scenario}") - inferred, inferred_result = checker.get_inferred_result( + inferred, inferred_result, inferred_valid = checker.get_inferred_result( scenario, effective_scenario, performance_result, mlperf_log, config, False) if has_power: From 0c135a009c9921a72671197b91ff46f0aefd52b1 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 17 Oct 2025 16:58:18 +0530 Subject: [PATCH 07/12] test commit --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 926c4174d..db7b94980 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -78,7 +78,7 @@ jobs: mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v5.1 --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi From 7d3c1dd2d944e5d7a255a135d555e88812d33286 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:32:03 +0530 Subject: [PATCH 08/12] Add extra args to skip sdxl images check for submission checker in workflow --- .../test-mlperf-inference-submission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index db7b94980..65af38a4a 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -75,11 +75,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v5.1 --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v5.1 --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags fi echo "::endgroup::" From e9aded589e388aaf081158123fa378ffb266061c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:51:19 +0530 Subject: [PATCH 09/12] Update MLPerf round version to 5.1 --- .../test-mlperf-inference-submission-generation.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 65af38a4a..71979a400 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -29,7 +29,7 @@ jobs: python-version: [ "3.12" ] division: ["closed", "open", "closed-open"] category: ["datacenter", "edge"] - round: ["6.0"] + round: ["5.1"] folder_name: ["closed", "open"] action: ["run", "docker"] exclude: @@ -75,11 +75,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v5.1 --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags fi echo "::endgroup::" From 8885b7fc29a1930f1c73200ed141ecbbbdb05c6d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:55:45 +0530 Subject: [PATCH 10/12] Add skip-extra-accuracy-files-check argument Added extra arguments to skip accuracy files check in MLPerf submission scripts. --- .../test-mlperf-inference-submission-generation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 71979a400..131c63854 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -75,11 +75,11 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags else - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags fi echo "::endgroup::" From 5d2e1f18c343b51b00fb4ac7af5ed72cb02ee9a6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 23 Oct 2025 12:01:10 +0530 Subject: [PATCH 11/12] test commit --- .../workflows/test-mlperf-inference-submission-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index 131c63854..c595d9f9b 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -75,7 +75,7 @@ jobs: fi if [ -n "${{ inputs.repo-url }}" ]; then - mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags else mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags From d672fbcdb6931ef060c04b0836485a5f5d40373c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 23 Oct 2025 12:07:46 +0530 Subject: [PATCH 12/12] test commit --- .../workflows/test-mlperf-inference-submission-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-submission-generation.yml b/.github/workflows/test-mlperf-inference-submission-generation.yml index c595d9f9b..c66beaafd 100644 --- a/.github/workflows/test-mlperf-inference-submission-generation.yml +++ b/.github/workflows/test-mlperf-inference-submission-generation.yml @@ -76,10 +76,10 @@ jobs: if [ -n "${{ inputs.repo-url }}" ]; then mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.${{ inputs.repo-url }},_branch.${{ inputs.ref }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags else mlc ${{ matrix.action }} script --tags=generate,inference,submission --adr.compiler.tags=gcc --version=v${{ matrix.round }} --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_examples/submission_round_${{ matrix.round }}/${{ matrix.folder_name }}/ --run_checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags - mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags + mlc ${{ matrix.action }} script --tags=run,inference,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/mlperf_submission --submission_tar_file=mysubmission.tar.gz --quiet --src_version=v${{ matrix.round }} --adr.inference-src.tags=_repo.https://github.com/mlcommons/inference,_branch.submission_dir --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check" $docker_tags fi echo "::endgroup::"