From 83e22297c6c8ff2f27625beb12f7c5e9b73d0e63 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 3 Nov 2025 11:39:10 +0100 Subject: [PATCH 01/21] first commit --- .github/workflows/test.yml | 2 ++ LICENSE | 2 +- computer_vision/object_detection/yolo_v5/run.py | 2 +- computer_vision/object_detection/yolo_v8/run.py | 12 ++++++------ .../bert_large/run_mlperf.py | 2 +- recommendation/dlrm/run.py | 2 +- requirements.txt | 1 + setup_deb.sh | 13 ++++++++++--- tests/test_pytorch_models.py | 2 +- utils/cv/pre_processing.py | 2 +- 10 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2921a548..03b62eaf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,6 +115,7 @@ jobs: COCO_IMG_PATH: aio_objdet_dataset COCO_ANNO_PATH: aio_objdet_dataset/annotations.json OMP_NUM_THREADS: 32 + AIO_NUM_THREADS: 32 S3_URL_CRITEO_DATASET: ${{ secrets.S3_URL_CRITEO_DATASET }} S3_URL_RESNET_50_V15_TF_FP32: ${{ secrets.S3_URL_RESNET_50_V15_TF_FP32 }} S3_URL_SSD_INCEPTION_V2_TF_FP32: ${{ secrets.S3_URL_SSD_INCEPTION_V2_TF_FP32 }} @@ -220,6 +221,7 @@ jobs: PYTHONPATH: ./ COCO_IMG_PATH: aio_objdet_dataset COCO_ANNO_PATH: aio_objdet_dataset/annotations.json + OMP_NUM_THREADS: 32 AIO_NUM_THREADS: 32 AIO_DEBUG_MODE: 0 S3_URL_CRITEO_DATASET: ${{ secrets.S3_URL_CRITEO_DATASET }} diff --git a/LICENSE b/LICENSE index 8580f840..42a38322 100644 --- a/LICENSE +++ b/LICENSE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright (c) 2024, Ampere Computing LLC + Copyright (c) 2025, Ampere Computing LLC Copyright (c) 2022 Andrej Karpathy Copyright (c) 2022 OpenAI Copyright (c) 2022 Stability AI diff --git a/computer_vision/object_detection/yolo_v5/run.py b/computer_vision/object_detection/yolo_v5/run.py index 945727fd..dd8d1828 100644 --- a/computer_vision/object_detection/yolo_v5/run.py +++ b/computer_vision/object_detection/yolo_v5/run.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC try: from utils import misc # noqa except ModuleNotFoundError: diff --git a/computer_vision/object_detection/yolo_v8/run.py b/computer_vision/object_detection/yolo_v8/run.py index 7df1d629..bbd51c24 100644 --- a/computer_vision/object_detection/yolo_v8/run.py +++ b/computer_vision/object_detection/yolo_v8/run.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC try: from utils import misc # noqa except ModuleNotFoundError: @@ -61,7 +61,7 @@ def run_ort_fp32(model_path, batch_size, num_runs, timeout, images_path, anno_pa # Ultralytics sets it to True by default. This way we suppress the logging by default while still allowing the user # to set it to True if needed from utils.ort import OrtRunner - from ultralytics.yolo.utils import ops + from ultralytics.utils import nms def run_single_pass(ort_runner, coco): shape = (640, 640) @@ -69,7 +69,7 @@ def run_single_pass(ort_runner, coco): output = ort_runner.run(batch_size) output = torch.from_numpy(output[0]) - output = ops.non_max_suppression(output) + output = nms.non_max_suppression(output) for i in range(batch_size): for d in range(output[i].shape[0]): @@ -97,11 +97,11 @@ def run_pytorch_fp(model_path, batch_size, num_runs, timeout, images_path, anno_ # Ultralytics sets it to True by default. This way we suppress the logging by default while still allowing the user # to set it to True if needed from utils.pytorch import PyTorchRunner - from ultralytics.yolo.utils import ops + from ultralytics.utils import nms def run_single_pass(pytorch_runner, coco): output = pytorch_runner.run(batch_size, coco.get_input_array((640, 640))) - output = ops.non_max_suppression(output) + output = nms.non_max_suppression(output) for i in range(batch_size): for d in range(output[i].shape[0]): @@ -121,7 +121,7 @@ def run_single_pass(pytorch_runner, coco): runner = PyTorchRunner(torch.jit.load(torchscript_model), disable_jit_freeze=disable_jit_freeze, - example_inputs=torch.stack(dataset.get_input_array((640, 640)))) + example_inputs=torch.stack((dataset.get_input_array((640, 640)),))) return run_model(run_single_pass, runner, dataset, batch_size, num_runs, timeout) diff --git a/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py b/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py index 57130f6c..284cd47f 100644 --- a/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py +++ b/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC try: from utils import misc # noqa except ModuleNotFoundError: diff --git a/recommendation/dlrm/run.py b/recommendation/dlrm/run.py index 97ce3a19..5997e085 100644 --- a/recommendation/dlrm/run.py +++ b/recommendation/dlrm/run.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC try: from utils import misc # noqa except ModuleNotFoundError: diff --git a/requirements.txt b/requirements.txt index 25e13945..1cb201c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ tiktoken ultralytics evaluate datasets +datasets[audio] soundfile librosa numba diff --git a/setup_deb.sh b/setup_deb.sh index 2e6b4a63..038315f8 100644 --- a/setup_deb.sh +++ b/setup_deb.sh @@ -4,6 +4,9 @@ set -eo pipefail +ln -fs /usr/share/zoneinfo/Europe/Warsaw /etc/localtime +echo "Europe/Warsaw" | tee /etc/timezone >/dev/null + log() { COLOR_DEFAULT='\033[0m' COLOR_CYAN='\033[1;36m' @@ -46,7 +49,7 @@ fi log "Installing system dependencies ..." sleep 1 apt-get update -y -apt-get install -y build-essential ffmpeg libsm6 libxext6 wget git unzip numactl libhdf5-dev cmake +apt-get install -y build-essential libsm6 libxext6 wget git unzip numactl libhdf5-dev cmake if ! python3 -c ""; then apt-get install -y python3 python3-pip fi @@ -76,8 +79,9 @@ sleep 1 ARCH=$ARCH python3 "$SCRIPT_DIR"/utils/setup/install_frameworks.py # get almost all python deps -pip3 install --break-system-packages -r "$(dirname "$0")/requirements.txt" || - pip3 install -r "$(dirname "$0")/requirements.txt" +PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --ignore-installed --upgrade pip +python3 -m pip install --break-system-packages -r "$(dirname "$0")/requirements.txt" || + python3 -m pip3 install -r "$(dirname "$0")/requirements.txt" apt install -y autoconf autogen automake build-essential libasound2-dev \ libflac-dev libogg-dev libtool libvorbis-dev libopus-dev libmp3lame-dev \ @@ -98,6 +102,9 @@ if [ "$(python3 -c 'import torch; print(torch.cuda.is_available())')" == "True" fi log "done.\n" +apt-get update -y +apt-get install -y ffmpeg + if [ -f "/etc/machine-id" ]; then cat /etc/machine-id >"$SCRIPT_DIR"/.setup_completed else diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 60b99472..6c260964 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC import os import signal import time diff --git a/utils/cv/pre_processing.py b/utils/cv/pre_processing.py index 7d452069..ae17a4b1 100644 --- a/utils/cv/pre_processing.py +++ b/utils/cv/pre_processing.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC import numpy as np import utils.misc as utils From 2fd04e2584f6df98859afa5beebef6a07db41f80 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 3 Nov 2025 14:18:01 +0100 Subject: [PATCH 02/21] wip --- setup_deb.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup_deb.sh b/setup_deb.sh index 038315f8..4ec5a5ec 100644 --- a/setup_deb.sh +++ b/setup_deb.sh @@ -4,9 +4,6 @@ set -eo pipefail -ln -fs /usr/share/zoneinfo/Europe/Warsaw /etc/localtime -echo "Europe/Warsaw" | tee /etc/timezone >/dev/null - log() { COLOR_DEFAULT='\033[0m' COLOR_CYAN='\033[1;36m' From 88a11619fae063a7dcc02a423669acbb66672362 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 3 Nov 2025 14:26:17 +0100 Subject: [PATCH 03/21] wip --- tests/test_pytorch_models.py | 248 +++++++++++++++++------------------ 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 6c260964..cc242d92 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -48,23 +48,23 @@ def wrapper(**kwargs): self.wrapper = wrapper - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_7b(self): - f1_ref = 0.330 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_13b(self): - f1_ref = 0.261 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_llama2_7b(self): + # f1_ref = 0.330 + # acc = run_process(self.wrapper, + # {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) + # + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_llama2_13b(self): + # f1_ref = 0.261 + # acc = run_process(self.wrapper, + # {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): @@ -85,19 +85,19 @@ def setUp(self): subprocess.run("rm /tmp/alpaca_recovered.tar.gz".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_alpaca(self): - from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - exact_match_ref, f1_ref = 0.220, 0.547 - acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_alpaca(self): + # from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # exact_match_ref, f1_ref = 0.220, 0.547 + # acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) class Whisper(unittest.TestCase): @@ -113,25 +113,25 @@ def wrapper_hf(**kwargs): self.wrapper_openai = wrapper_openai self.wrapper_hf = wrapper_hf - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - def test_whisper_tiny_en(self): - wer_ref = 0.155 - acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - def test_whisper_hf_tiny_en(self): - wer_ref = 0.111 - acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, - "batch_size": 4, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_large(self): - wer_ref = 0.124 - acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + # def test_whisper_tiny_en(self): + # wer_ref = 0.155 + # acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + # def test_whisper_hf_tiny_en(self): + # wer_ref = 0.111 + # acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, + # "batch_size": 4, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_large(self): + # wer_ref = 0.124 + # acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) class WhisperTranslate(unittest.TestCase): @@ -156,13 +156,13 @@ def wrapper(**kwargs): self.wrapper = wrapper - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_translate_medium(self): - wer_ref = 0.475 - acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, - "dataset_path": self.dataset_path}) - self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_translate_medium(self): + # wer_ref = 0.475 + # acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, + # "dataset_path": self.dataset_path}) + # self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) class DLRM(unittest.TestCase): @@ -184,17 +184,17 @@ def setUp(self): f"{'https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - def test_dlrm_debug(self): - from recommendation.dlrm.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - auc_ref = 0.583 - acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, - "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) - self.assertTrue(acc["auc"] / auc_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # def test_dlrm_debug(self): + # from recommendation.dlrm.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # auc_ref = 0.583 + # acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + # "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) + # self.assertTrue(acc["auc"] / auc_ref > 0.95) class BERT(unittest.TestCase): @@ -214,17 +214,17 @@ def setUp(self): f"{'https://zenodo.org/records/3733896/files/model.pytorch?download=1'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - def test_bert_large_mlperf(self): - from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - exact_match_ref, f1_ref = 0.750, 0.817 - acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, - "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) - self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # def test_bert_large_mlperf(self): + # from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # exact_match_ref, f1_ref = 0.750, 0.817 + # acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, + # "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) + # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) def download_imagenet_maybe(): @@ -251,36 +251,36 @@ class DenseNet(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_densenet_121(self): - from computer_vision.classification.densenet_121.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.717, 0.905 - acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - "disable_jit_freeze": False}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_densenet_121(self): + # from computer_vision.classification.densenet_121.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.717, 0.905 + # acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "disable_jit_freeze": False}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class Inception(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_inception_v3(self): - from computer_vision.classification.inception_v3.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.765, 0.932 - acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - "disable_jit_freeze": False}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_inception_v3(self): + # from computer_vision.classification.inception_v3.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.765, 0.932 + # acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "disable_jit_freeze": False}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class ResNet(unittest.TestCase): @@ -304,17 +304,17 @@ class VGG(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_vgg16(self): - from computer_vision.classification.vgg_16.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.661, 0.896 - acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_vgg16(self): + # from computer_vision.classification.vgg_16.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.661, 0.896 + # acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) def download_coco_maybe(): @@ -365,17 +365,17 @@ def setUp(self): # "timeout": None, "disable_jit_freeze": False}) # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) - def test_yolo_v8_s(self): - from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - coco_map_ref = 0.353 - acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, - "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, - "timeout": None, "disable_jit_freeze": False}) - self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) + # def test_yolo_v8_s(self): + # from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # coco_map_ref = 0.353 + # acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, + # "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, + # "timeout": None, "disable_jit_freeze": False}) + # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) if __name__ == "__main__": From 0a542cd6f5c8b7b9645a25b0bd75b432e7fd326e Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 3 Nov 2025 15:57:22 +0100 Subject: [PATCH 04/21] wip --- tests/test_pytorch_models.py | 248 +++++++++++++++++------------------ 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index cc242d92..6c260964 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -48,23 +48,23 @@ def wrapper(**kwargs): self.wrapper = wrapper - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_llama2_7b(self): - # f1_ref = 0.330 - # acc = run_process(self.wrapper, - # {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) - # - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_llama2_13b(self): - # f1_ref = 0.261 - # acc = run_process(self.wrapper, - # {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_7b(self): + f1_ref = 0.330 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_13b(self): + f1_ref = 0.261 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): @@ -85,19 +85,19 @@ def setUp(self): subprocess.run("rm /tmp/alpaca_recovered.tar.gz".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_alpaca(self): - # from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # exact_match_ref, f1_ref = 0.220, 0.547 - # acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_alpaca(self): + from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + exact_match_ref, f1_ref = 0.220, 0.547 + acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + self.assertTrue(acc["f1"] / f1_ref > 0.95) class Whisper(unittest.TestCase): @@ -113,25 +113,25 @@ def wrapper_hf(**kwargs): self.wrapper_openai = wrapper_openai self.wrapper_hf = wrapper_hf - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - # def test_whisper_tiny_en(self): - # wer_ref = 0.155 - # acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) - # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - # - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - # def test_whisper_hf_tiny_en(self): - # wer_ref = 0.111 - # acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, - # "batch_size": 4, "timeout": None}) - # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - # - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - # def test_whisper_large(self): - # wer_ref = 0.124 - # acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) - # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + def test_whisper_tiny_en(self): + wer_ref = 0.155 + acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) + self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + def test_whisper_hf_tiny_en(self): + wer_ref = 0.111 + acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, + "batch_size": 4, "timeout": None}) + self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + def test_whisper_large(self): + wer_ref = 0.124 + acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) + self.assertTrue(wer_ref / acc["wer_score"] > 0.95) class WhisperTranslate(unittest.TestCase): @@ -156,13 +156,13 @@ def wrapper(**kwargs): self.wrapper = wrapper - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - # def test_whisper_translate_medium(self): - # wer_ref = 0.475 - # acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, - # "dataset_path": self.dataset_path}) - # self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + def test_whisper_translate_medium(self): + wer_ref = 0.475 + acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, + "dataset_path": self.dataset_path}) + self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) class DLRM(unittest.TestCase): @@ -184,17 +184,17 @@ def setUp(self): f"{'https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # def test_dlrm_debug(self): - # from recommendation.dlrm.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # auc_ref = 0.583 - # acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, - # "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) - # self.assertTrue(acc["auc"] / auc_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + def test_dlrm_debug(self): + from recommendation.dlrm.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + auc_ref = 0.583 + acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) + self.assertTrue(acc["auc"] / auc_ref > 0.95) class BERT(unittest.TestCase): @@ -214,17 +214,17 @@ def setUp(self): f"{'https://zenodo.org/records/3733896/files/model.pytorch?download=1'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # def test_bert_large_mlperf(self): - # from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # exact_match_ref, f1_ref = 0.750, 0.817 - # acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, - # "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) - # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + def test_bert_large_mlperf(self): + from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + exact_match_ref, f1_ref = 0.750, 0.817 + acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, + "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) + self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + self.assertTrue(acc["f1"] / f1_ref > 0.95) def download_imagenet_maybe(): @@ -251,36 +251,36 @@ class DenseNet(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_densenet_121(self): - # from computer_vision.classification.densenet_121.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.717, 0.905 - # acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - # "disable_jit_freeze": False}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_densenet_121(self): + from computer_vision.classification.densenet_121.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.717, 0.905 + acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + "disable_jit_freeze": False}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class Inception(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_inception_v3(self): - # from computer_vision.classification.inception_v3.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.765, 0.932 - # acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - # "disable_jit_freeze": False}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_inception_v3(self): + from computer_vision.classification.inception_v3.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.765, 0.932 + acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + "disable_jit_freeze": False}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class ResNet(unittest.TestCase): @@ -304,17 +304,17 @@ class VGG(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_vgg16(self): - # from computer_vision.classification.vgg_16.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.661, 0.896 - # acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_vgg16(self): + from computer_vision.classification.vgg_16.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.661, 0.896 + acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) def download_coco_maybe(): @@ -365,17 +365,17 @@ def setUp(self): # "timeout": None, "disable_jit_freeze": False}) # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) - # def test_yolo_v8_s(self): - # from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # coco_map_ref = 0.353 - # acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, - # "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, - # "timeout": None, "disable_jit_freeze": False}) - # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) + def test_yolo_v8_s(self): + from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + coco_map_ref = 0.353 + acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, + "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, + "timeout": None, "disable_jit_freeze": False}) + self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) if __name__ == "__main__": From 447c48c160a7f88eb1f48bf043460ccf01341b12 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 3 Nov 2025 16:53:27 +0100 Subject: [PATCH 05/21] wip --- tests/test_pytorch_models.py | 244 +++++++++++++++++------------------ 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 6c260964..9cd04725 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -48,23 +48,23 @@ def wrapper(**kwargs): self.wrapper = wrapper - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_7b(self): - f1_ref = 0.330 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_13b(self): - f1_ref = 0.261 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_llama2_7b(self): + # f1_ref = 0.330 + # acc = run_process(self.wrapper, + # {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) + # + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_llama2_13b(self): + # f1_ref = 0.261 + # acc = run_process(self.wrapper, + # {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): @@ -85,19 +85,19 @@ def setUp(self): subprocess.run("rm /tmp/alpaca_recovered.tar.gz".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_alpaca(self): - from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - exact_match_ref, f1_ref = 0.220, 0.547 - acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + # def test_alpaca(self): + # from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # exact_match_ref, f1_ref = 0.220, 0.547 + # acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, + # "timeout": None, "dataset_path": self.dataset_path}) + # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) class Whisper(unittest.TestCase): @@ -113,25 +113,25 @@ def wrapper_hf(**kwargs): self.wrapper_openai = wrapper_openai self.wrapper_hf = wrapper_hf - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - def test_whisper_tiny_en(self): - wer_ref = 0.155 - acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + # def test_whisper_tiny_en(self): + # wer_ref = 0.155 + # acc = run_process(self.wrapper_openai, {"model_name": "tiny.en", "num_runs": 30, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") - def test_whisper_hf_tiny_en(self): - wer_ref = 0.111 - acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, - "batch_size": 4, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 50, "too little memory") + # def test_whisper_hf_tiny_en(self): + # wer_ref = 0.111 + # acc = run_process(self.wrapper_hf, {"model_name": "openai/whisper-tiny.en", "num_runs": 18, + # "batch_size": 4, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_large(self): - wer_ref = 0.124 - acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_large(self): + # wer_ref = 0.124 + # acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) class WhisperTranslate(unittest.TestCase): @@ -156,13 +156,13 @@ def wrapper(**kwargs): self.wrapper = wrapper - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_translate_medium(self): - wer_ref = 0.475 - acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, - "dataset_path": self.dataset_path}) - self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_translate_medium(self): + # wer_ref = 0.475 + # acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, + # "dataset_path": self.dataset_path}) + # self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) class DLRM(unittest.TestCase): @@ -184,17 +184,17 @@ def setUp(self): f"{'https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - def test_dlrm_debug(self): - from recommendation.dlrm.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - auc_ref = 0.583 - acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, - "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) - self.assertTrue(acc["auc"] / auc_ref > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # def test_dlrm_debug(self): + # from recommendation.dlrm.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # auc_ref = 0.583 + # acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + # "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) + # self.assertTrue(acc["auc"] / auc_ref > 0.95) class BERT(unittest.TestCase): @@ -214,17 +214,17 @@ def setUp(self): f"{'https://zenodo.org/records/3733896/files/model.pytorch?download=1'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - def test_bert_large_mlperf(self): - from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - exact_match_ref, f1_ref = 0.750, 0.817 - acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, - "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) - self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - self.assertTrue(acc["f1"] / f1_ref > 0.95) + # def test_bert_large_mlperf(self): + # from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # exact_match_ref, f1_ref = 0.750, 0.817 + # acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, + # "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) + # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + # self.assertTrue(acc["f1"] / f1_ref > 0.95) def download_imagenet_maybe(): @@ -251,36 +251,36 @@ class DenseNet(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_densenet_121(self): - from computer_vision.classification.densenet_121.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.717, 0.905 - acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - "disable_jit_freeze": False}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_densenet_121(self): + # from computer_vision.classification.densenet_121.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.717, 0.905 + # acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "disable_jit_freeze": False}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class Inception(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_inception_v3(self): - from computer_vision.classification.inception_v3.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.765, 0.932 - acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, - "disable_jit_freeze": False}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_inception_v3(self): + # from computer_vision.classification.inception_v3.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.765, 0.932 + # acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "disable_jit_freeze": False}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class ResNet(unittest.TestCase): @@ -304,17 +304,17 @@ class VGG(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - def test_vgg16(self): - from computer_vision.classification.vgg_16.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - top_1_ref, top_5_ref = 0.661, 0.896 - acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, - "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) - self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + # def test_vgg16(self): + # from computer_vision.classification.vgg_16.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # top_1_ref, top_5_ref = 0.661, 0.896 + # acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) + # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) def download_coco_maybe(): @@ -365,17 +365,17 @@ def setUp(self): # "timeout": None, "disable_jit_freeze": False}) # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) - def test_yolo_v8_s(self): - from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - coco_map_ref = 0.353 - acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, - "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, - "timeout": None, "disable_jit_freeze": False}) - self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) + # def test_yolo_v8_s(self): + # from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 + # + # def wrapper(**kwargs): + # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + # + # coco_map_ref = 0.353 + # acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, + # "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, + # "timeout": None, "disable_jit_freeze": False}) + # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) if __name__ == "__main__": From 13538c37eb3095a4eceddada42f682e93d48b5e7 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 10:42:55 +0100 Subject: [PATCH 06/21] wip --- tests/test_pytorch_models.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 9cd04725..ffcf5e6c 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -259,7 +259,8 @@ def setUp(self): # # top_1_ref, top_5_ref = 0.717, 0.905 # acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + # "timeout": None, # "disable_jit_freeze": False}) # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) @@ -277,7 +278,8 @@ def setUp(self): # # top_1_ref, top_5_ref = 0.765, 0.932 # acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None, + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + # "timeout": None, # "disable_jit_freeze": False}) # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) @@ -312,7 +314,8 @@ def setUp(self): # # top_1_ref, top_5_ref = 0.661, 0.896 # acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, "timeout": None}) + # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + # "timeout": None}) # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) @@ -365,17 +368,17 @@ def setUp(self): # "timeout": None, "disable_jit_freeze": False}) # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) - # def test_yolo_v8_s(self): - # from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # coco_map_ref = 0.353 - # acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, - # "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, - # "timeout": None, "disable_jit_freeze": False}) - # self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) + def test_yolo_v8_s(self): + from computer_vision.object_detection.yolo_v8.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + coco_map_ref = 0.353 + acc = run_process(wrapper, {"model_path": self.yolo_v8_s_path, "images_path": self.dataset_path, + "anno_path": self.annotations_path, "batch_size": 1, "num_runs": 465, + "timeout": None, "disable_jit_freeze": False}) + self.assertTrue(acc["coco_map"] / coco_map_ref > 0.95) if __name__ == "__main__": From e26dc0d120b2fe9e494ea10a7e2c6ceb1a514d6d Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 12:58:17 +0100 Subject: [PATCH 07/21] wip --- tests/test_pytorch_models.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index ffcf5e6c..15a2a375 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -306,18 +306,18 @@ class VGG(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_vgg16(self): - # from computer_vision.classification.vgg_16.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.661, 0.896 - # acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, - # "timeout": None}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_vgg16(self): + from computer_vision.classification.vgg_16.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.661, 0.896 + acc = run_process(wrapper, {"model_name": "vgg16", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + "timeout": None}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) def download_coco_maybe(): From 48fe49745a30b08adbba43e3bdeb4fe08e0c2766 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 13:19:07 +0100 Subject: [PATCH 08/21] wip --- tests/test_pytorch_models.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 15a2a375..de7c7925 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -270,19 +270,19 @@ class Inception(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_inception_v3(self): - # from computer_vision.classification.inception_v3.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.765, 0.932 - # acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, - # "timeout": None, - # "disable_jit_freeze": False}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_inception_v3(self): + from computer_vision.classification.inception_v3.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.765, 0.932 + acc = run_process(wrapper, {"model_name": "inception_v3", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + "timeout": None, + "disable_jit_freeze": False}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class ResNet(unittest.TestCase): From 41126578b6cd6930f972e6e39f2139504c9159a5 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 13:55:05 +0100 Subject: [PATCH 09/21] wip --- tests/test_pytorch_models.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index de7c7925..4b5e91dd 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -251,19 +251,19 @@ class DenseNet(unittest.TestCase): def setUp(self): self.dataset_path, self.labels_path = download_imagenet_maybe() - # def test_densenet_121(self): - # from computer_vision.classification.densenet_121.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # top_1_ref, top_5_ref = 0.717, 0.905 - # acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, - # "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, - # "timeout": None, - # "disable_jit_freeze": False}) - # self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) - # self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) + def test_densenet_121(self): + from computer_vision.classification.densenet_121.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + top_1_ref, top_5_ref = 0.717, 0.905 + acc = run_process(wrapper, {"model_name": "densenet121", "images_path": self.dataset_path, + "labels_path": self.labels_path, "batch_size": 32, "num_runs": 10, + "timeout": None, + "disable_jit_freeze": False}) + self.assertTrue(acc["top_1_acc"] / top_1_ref > 0.95) + self.assertTrue(acc["top_5_acc"] / top_5_ref > 0.95) class Inception(unittest.TestCase): From d9ce0afaf58160d3b9a11a4b665d675c454aa42d Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 15:55:18 +0100 Subject: [PATCH 10/21] wip --- .github/workflows/test.yml | 2 +- tests/test_pytorch_models.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03b62eaf..7a3722da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -249,7 +249,7 @@ jobs: - name: benchmark.py test run: | - { echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; } | PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py + # { echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; } | PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py # testing second time to ensure that left-over files don't interrupt, etc. - this time no-interactive mode PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py --no-interactive --memory 30 --max-threads 24 diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 4b5e91dd..2b1253d0 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -214,17 +214,17 @@ def setUp(self): f"{'https://zenodo.org/records/3733896/files/model.pytorch?download=1'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # def test_bert_large_mlperf(self): - # from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # exact_match_ref, f1_ref = 0.750, 0.817 - # acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, - # "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) - # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + def test_bert_large_mlperf(self): + from natural_language_processing.extractive_question_answering.bert_large.run_mlperf import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + exact_match_ref, f1_ref = 0.750, 0.817 + acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, + "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) + self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + self.assertTrue(acc["f1"] / f1_ref > 0.95) def download_imagenet_maybe(): From dfe80294a9f60fcbd5b4c5918202fcba7239ea10 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 16:38:16 +0100 Subject: [PATCH 11/21] wip --- tests/test_pytorch_models.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 2b1253d0..90bacac8 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -184,17 +184,17 @@ def setUp(self): f"{'https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt'}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # def test_dlrm_debug(self): - # from recommendation.dlrm.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # auc_ref = 0.583 - # acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, - # "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) - # self.assertTrue(acc["auc"] / auc_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + def test_dlrm_debug(self): + from recommendation.dlrm.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + auc_ref = 0.583 + acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + "batch_size": 2048, "num_runs": 30, "timeout": None, "debug": True}) + self.assertTrue(acc["auc"] / auc_ref > 0.95) class BERT(unittest.TestCase): From 9bfadb9f8e3926d9a3de2fcdd8abfdf2d87a5158 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 4 Nov 2025 17:18:37 +0100 Subject: [PATCH 12/21] wip --- tests/test_pytorch_models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 90bacac8..bdf332bc 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -156,13 +156,13 @@ def wrapper(**kwargs): self.wrapper = wrapper - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - # def test_whisper_translate_medium(self): - # wer_ref = 0.475 - # acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, - # "dataset_path": self.dataset_path}) - # self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + def test_whisper_translate_medium(self): + wer_ref = 0.475 + acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, + "dataset_path": self.dataset_path}) + self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) class DLRM(unittest.TestCase): From 435dfa06ef8243b3198506f463b579fe149e5d80 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 5 Nov 2025 09:35:52 +0100 Subject: [PATCH 13/21] wip --- tests/test_pytorch_models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index bdf332bc..90bacac8 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -156,13 +156,13 @@ def wrapper(**kwargs): self.wrapper = wrapper - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_translate_medium(self): - wer_ref = 0.475 - acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, - "dataset_path": self.dataset_path}) - self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_translate_medium(self): + # wer_ref = 0.475 + # acc = run_process(self.wrapper, {"model_name": "large", "num_runs": 30, "timeout": None, + # "dataset_path": self.dataset_path}) + # self.assertTrue(wer_ref / acc["bleu_score"] > 0.95) class DLRM(unittest.TestCase): From 8da61dca901495f272b0258948291adcfd618e77 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 5 Nov 2025 11:11:54 +0100 Subject: [PATCH 14/21] wip --- tests/test_pytorch_models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 90bacac8..c7afefd9 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -126,12 +126,12 @@ def wrapper_hf(**kwargs): # "batch_size": 4, "timeout": None}) # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - # def test_whisper_large(self): - # wer_ref = 0.124 - # acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) - # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + def test_whisper_large(self): + wer_ref = 0.124 + acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) + self.assertTrue(wer_ref / acc["wer_score"] > 0.95) class WhisperTranslate(unittest.TestCase): From 80f1a6095bf586c8ed60ff1ef1454bb9d74c40c3 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 5 Nov 2025 11:59:44 +0100 Subject: [PATCH 15/21] wip --- tests/test_pytorch_models.py | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index c7afefd9..844a9041 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -85,19 +85,19 @@ def setUp(self): subprocess.run("rm /tmp/alpaca_recovered.tar.gz".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_alpaca(self): - # from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 - # - # def wrapper(**kwargs): - # kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - # - # exact_match_ref, f1_ref = 0.220, 0.547 - # acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_alpaca(self): + from natural_language_processing.text_generation.alpaca.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + exact_match_ref, f1_ref = 0.220, 0.547 + acc = run_process(wrapper, {"model_path": self.model_path, "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) + self.assertTrue(acc["f1"] / f1_ref > 0.95) class Whisper(unittest.TestCase): @@ -126,12 +126,12 @@ def wrapper_hf(**kwargs): # "batch_size": 4, "timeout": None}) # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") - def test_whisper_large(self): - wer_ref = 0.124 - acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) - self.assertTrue(wer_ref / acc["wer_score"] > 0.95) + # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "too slow to run with native") + # def test_whisper_large(self): + # wer_ref = 0.124 + # acc = run_process(self.wrapper_openai, {"model_name": "large", "num_runs": 30, "timeout": None}) + # self.assertTrue(wer_ref / acc["wer_score"] > 0.95) class WhisperTranslate(unittest.TestCase): From 0a9b49b869cdbb47cd24e03852928da519bd4f48 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 5 Nov 2025 13:10:39 +0100 Subject: [PATCH 16/21] wip --- tests/test_pytorch_models.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 844a9041..cdfda02a 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -48,23 +48,23 @@ def wrapper(**kwargs): self.wrapper = wrapper - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_llama2_7b(self): - # f1_ref = 0.330 - # acc = run_process(self.wrapper, - # {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) - # - # @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") - # @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - # def test_llama2_13b(self): - # f1_ref = 0.261 - # acc = run_process(self.wrapper, - # {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, - # "timeout": None, "dataset_path": self.dataset_path}) - # self.assertTrue(acc["f1"] / f1_ref > 0.95) + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_7b(self): + f1_ref = 0.330 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_13b(self): + f1_ref = 0.261 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): From 062b51be697b9d1e9fd57b3ce9dfd06c31c2e561 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 5 Nov 2025 15:32:01 +0100 Subject: [PATCH 17/21] wip --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1cb201c5..93b1bcaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ sentencepiece tiktoken ultralytics evaluate -datasets +datasets>=2.19 datasets[audio] soundfile librosa From d717990bb723baa0208e8c35ab234364b9011e74 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 6 Nov 2025 15:53:28 +0100 Subject: [PATCH 18/21] wip --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a3722da..8e303161 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -265,7 +265,7 @@ jobs: IGNORE_DATASET_LIMITS=1 AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/classification/resnet_50_v15/run.py -m resnet50 -p fp32 -b 16 -f pytorch - AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 speech_recognition/whisper/run.py -m tiny.en + # AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 speech_recognition/whisper/run.py -m tiny.en IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/mobilenet_v2/run.py -p fp32 -f pytorch --timeout=60 From a36b050533af5febfca9e05dfc784ffd9a6b4c25 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 6 Nov 2025 17:08:14 +0100 Subject: [PATCH 19/21] wip --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8e303161..36695a38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -249,7 +249,7 @@ jobs: - name: benchmark.py test run: | - # { echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; } | PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py + { echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; } | PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py # testing second time to ensure that left-over files don't interrupt, etc. - this time no-interactive mode PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py --no-interactive --memory 30 --max-threads 24 From f4fa41728ab060c7db0ad22ad20d8760e11da728 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 7 Nov 2025 17:48:06 +0100 Subject: [PATCH 20/21] wip --- benchmark.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark.py b/benchmark.py index d7fa46bc..8fc2bf31 100644 --- a/benchmark.py +++ b/benchmark.py @@ -15,8 +15,8 @@ "ResNet-50 v1.5": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40resnet_50_v1.5.json", # noqa "YOLO v8s": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40yolo_v8_s.json", # noqa "BERT large": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40bert_large_mlperf_squad.json", # noqa - "DLRM": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40dlrm_torchbench.json", # noqa - "Whisper medium EN": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40whisper_medium.en.json" # noqa + "DLRM": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40dlrm_torchbench.json" # noqa + # "Whisper medium EN": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/q80_30%40ampere_pytorch_1.10.0%40whisper_medium.en.json" # noqa }, "Altra Max": { "ResNet-50 v1.5": "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/m128_30%40ampere_pytorch_1.10.0%40resnet_50_v1.5.json", # noqa @@ -676,7 +676,8 @@ def convert_name(text): def main(): - models = [ResNet50, YOLO, BERT, DLRM, Whisper] + # models = [ResNet50, YOLO, BERT, DLRM, Whisper] + models = [ResNet50, YOLO, BERT, DLRM] parser = argparse.ArgumentParser(prog="AML benchmarking tool") parser.add_argument("--no-interactive", action="store_true", help="don't ask for user input") parser.add_argument("--model", type=str, choices=[convert_name(model.model_name) for model in models], From b2f78810bb1a6e7f06eedc21f3220342488bcafe Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 7 Nov 2025 18:03:07 +0100 Subject: [PATCH 21/21] wip --- benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark.py b/benchmark.py index 8fc2bf31..ece156c2 100644 --- a/benchmark.py +++ b/benchmark.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2024, Ampere Computing LLC +# Copyright (c) 2025, Ampere Computing LLC + import os import sys import json