Skip to content

Commit d608bc8

Browse files
jan-grzybek-amperejan-grzybek-ampere
authored
General benchmark script (#228)
1 parent 5b39d49 commit d608bc8

File tree

21 files changed

+1467
-63
lines changed

21 files changed

+1467
-63
lines changed

‎.github/workflows/test.yml‎

Lines changed: 248 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,38 +4,273 @@ on:
44
branches: [ main ]
55
pull_request:
66
branches: [ main ]
7-
workflow_dispatch:
7+
8+
concurrency:
9+
group: ${{ github.head_ref }}
10+
cancel-in-progress: true
811

912
jobs:
10-
test_setup:
11-
runs-on: ubuntu-22.04
12-
name: Ubuntu 22.04 x86-64 AML test
13+
linter:
14+
runs-on: ubuntu-latest
15+
container: ubuntu:22.04
16+
name: Lint - Flake8
17+
steps:
18+
- name: Install git & flake8
19+
run:
20+
apt-get update && apt-get install -y git python3-pip && pip3 install flake8
21+
22+
- name: Git checkout
23+
uses: actions/checkout@v4
24+
with:
25+
submodules: false
26+
27+
- name: Lint with flake8
28+
run:
29+
python3 -m flake8 --max-complexity=10 --max-line-length=120 --statistics --exit-zero
30+
31+
test_x86:
32+
runs-on: ubuntu-latest
33+
container: ubuntu:22.04
34+
name: x86-64 - Ubuntu 22.04 - all frameworks (native)
35+
env:
36+
PYTHONPATH: ./
37+
COCO_IMG_PATH: aio_objdet_dataset
38+
COCO_ANNO_PATH: aio_objdet_dataset/annotations.json
39+
OMP_NUM_THREADS: 4
1340
steps:
41+
- name: Install git
42+
run:
43+
apt-get update && apt-get install -y git
44+
1445
- name: Git checkout & pull submodules
15-
uses: actions/checkout@v3
46+
uses: actions/checkout@v4
1647
with:
1748
submodules: true
1849

1950
- name: Set up AML
2051
run:
21-
sudo -s FORCE_INSTALL=1 bash setup_deb.sh
52+
FORCE_INSTALL=1 bash setup_deb.sh
2253

2354
- name: AML imports test
2455
run:
25-
sudo -s PYTHONPATH=./ python3 utils/tests/setup_test_utils/attempt_imports.py
56+
python3 utils/tests/setup_test_utils/attempt_imports.py
2657

2758
- name: AML smoke test
28-
run: |
59+
run: |
60+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/aio_objdet_dataset.tar.gz > /dev/null 2>&1
61+
tar -xf aio_objdet_dataset.tar.gz > /dev/null
62+
2963
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/resnet_50_v15_tf_fp32.pb > /dev/null 2>&1
30-
sudo -s PYTHONPATH=./ OMP_NUM_THREADS=2 python3 computer_vision/classification/resnet_50_v15/run.py -m resnet_50_v15_tf_fp32.pb -p fp32 -f tf --timeout=60
64+
python3 computer_vision/classification/resnet_50_v15/run.py -m resnet_50_v15_tf_fp32.pb -p fp32 -f tf --timeout=60
3165
32-
sudo -s PYTHONPATH=./ OMP_NUM_THREADS=2 python3 computer_vision/classification/mobilenet_v2/run.py -p fp32 -f pytorch --timeout=60
66+
python3 computer_vision/classification/mobilenet_v2/run.py -p fp32 -f pytorch --timeout=60
3367
68+
wget https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt > /dev/null 2>&1
69+
python3 computer_vision/object_detection/yolo_v8/run.py -m yolov8n.pt -f pytorch -p fp32 --timeout=60
70+
71+
python3 speech_recognition/whisper/run.py -m small.en
72+
73+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/ssd_inception_v2_tf_fp32.pb > /dev/null 2>&1
74+
python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -p fp32 --timeout=60
75+
76+
wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1
77+
python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort
78+
79+
wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1
80+
tar -xf vgg16.tar.gz > /dev/null
81+
python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort
82+
83+
test_arm64:
84+
runs-on: self-hosted
85+
container:
86+
image: ubuntu:22.04
87+
options: --memory=170g
88+
name: ARM64 - Ubuntu 22.04 - all frameworks (native)
89+
env:
90+
PYTHONPATH: ./
91+
COCO_IMG_PATH: aio_objdet_dataset
92+
COCO_ANNO_PATH: aio_objdet_dataset/annotations.json
93+
OMP_NUM_THREADS: 32
94+
steps:
95+
- name: Install git
96+
run:
97+
apt-get update && apt-get install -y git
98+
99+
- name: Git checkout & pull submodules
100+
uses: actions/checkout@v4
101+
with:
102+
submodules: true
103+
104+
- name: Set up AML
105+
run:
106+
bash setup_deb.sh
107+
108+
- name: AML smoke test
109+
run: |
34110
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/aio_objdet_dataset.tar.gz > /dev/null 2>&1
35-
tar -xvf aio_objdet_dataset.tar.gz > /dev/null 2>&1
111+
tar -xf aio_objdet_dataset.tar.gz > /dev/null
112+
113+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/resnet_50_v15_tf_fp32.pb > /dev/null 2>&1
114+
python3 computer_vision/classification/resnet_50_v15/run.py -m resnet_50_v15_tf_fp32.pb -p fp32 -f tf --timeout=60
115+
116+
python3 computer_vision/classification/mobilenet_v2/run.py -p fp32 -f pytorch --timeout=60
36117
37118
wget https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt > /dev/null 2>&1
38-
sudo -s PYTHONPATH=./ OMP_NUM_THREADS=2 python3 computer_vision/object_detection/yolo_v8/run.py -m yolov8n.pt -f pytorch -p fp32 --timeout=60 --images_path=aio_objdet_dataset --anno_path=aio_objdet_dataset/annotations.json
119+
python3 computer_vision/object_detection/yolo_v8/run.py -m yolov8n.pt -f pytorch -p fp32 --timeout=60
120+
121+
python3 speech_recognition/whisper/run.py -m small.en
122+
123+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/ssd_inception_v2_tf_fp32.pb > /dev/null 2>&1
124+
python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -p fp32 --timeout=60
125+
126+
wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1
127+
python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort
128+
129+
wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1
130+
tar -xf vgg16.tar.gz > /dev/null
131+
python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort
132+
133+
test_pytorch_arm64_sh:
134+
runs-on: self-hosted
135+
container:
136+
image: ubuntu:22.04
137+
options: --memory=170g
138+
name: Ampere Altra - Ampere optimized PyTorch (shell installer)
139+
steps:
140+
- name: Install Ampere optimized PyTorch
141+
run: |
142+
apt-get update && apt-get install -y wget
143+
bash -c "$(wget -qO- https://ampereaidevelopus.s3.amazonaws.com/releases/1.10.0/binaries/install_ampere_pytorch_u22_1_10_0.sh)"
144+
145+
- name: Git checkout & pull submodules
146+
uses: actions/checkout@v4
147+
with:
148+
submodules: true
149+
150+
- name: Set up AML
151+
run:
152+
bash setup_deb.sh
153+
154+
- name: benchmark.py test
155+
run: |
156+
PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py --no-interactive
157+
158+
test_pytorch_arm64_docker:
159+
runs-on: self-hosted
160+
container:
161+
image: amperecomputingai/pytorch:latest
162+
options: --memory=170g
163+
name: Ampere Altra - Ampere optimized PyTorch (Docker image)
164+
env:
165+
PYTHONPATH: ./
166+
COCO_IMG_PATH: aio_objdet_dataset
167+
COCO_ANNO_PATH: aio_objdet_dataset/annotations.json
168+
AIO_NUM_THREADS: 32
169+
HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }}
170+
steps:
171+
- name: Git checkout & pull submodules
172+
uses: actions/checkout@v4
173+
with:
174+
submodules: true
175+
176+
- name: Set up AML
177+
run: |
178+
bash setup_deb.sh
179+
echo $HF_HUB_TOKEN > ~/.cache/huggingface/token
180+
181+
- name: benchmark.py test
182+
run: |
183+
{ echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; echo "y"; } | PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py
184+
# testing second time to ensure that left-over files don't interrupt, etc. - this time no-interactive mode
185+
PYTHONPATH=/__w/ampere_model_library/ampere_model_library python3 benchmark.py --no-interactive --memory 30 --max-threads 24
186+
187+
- name: AML smoke test
188+
run: |
189+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/aio_objdet_dataset.tar.gz > /dev/null 2>&1
190+
tar -xf aio_objdet_dataset.tar.gz > /dev/null
191+
192+
wget https://ampereaimodelzoo.s3.amazonaws.com/alpaca_data.json > /dev/null 2>&1
193+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 natural_language_processing/text_generation/llama2/run.py -m meta-llama/Llama-2-7b-chat-hf --dataset_path=alpaca_data.json
194+
195+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 recommendation/dlrm_torchbench/run.py -p fp32
196+
197+
IGNORE_DATASET_LIMITS=1 AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/classification/resnet_50_v15/run.py -m resnet50 -p fp32 -b 16 -f pytorch
198+
199+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 speech_recognition/whisper/run.py -m tiny.en
200+
201+
python3 computer_vision/classification/mobilenet_v2/run.py -p fp32 -f pytorch --timeout=60
202+
203+
wget https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8l.pt > /dev/null 2>&1
204+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/object_detection/yolo_v8/run.py -m yolov8l.pt -p fp32 -f pytorch
205+
206+
wget -O bert_large_mlperf.pt https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/bert_large_pytorch_fp32.pytorch > /dev/null 2>&1
207+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py -m bert_large_mlperf.pt -p fp32 -f pytorch
208+
209+
test_tensorflow_arm64:
210+
runs-on: self-hosted
211+
container:
212+
image: amperecomputingai/tensorflow:latest
213+
options: --memory=170g
214+
name: Ampere Altra - Ampere optimized TensorFlow (Docker image)
215+
env:
216+
PYTHONPATH: ./
217+
COCO_IMG_PATH: aio_objdet_dataset
218+
COCO_ANNO_PATH: aio_objdet_dataset/annotations.json
219+
AIO_NUM_THREADS: 32
220+
HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }}
221+
steps:
222+
- name: Git checkout & pull submodules
223+
uses: actions/checkout@v4
224+
with:
225+
submodules: true
226+
227+
- name: Set up AML
228+
run: |
229+
bash setup_deb.sh
230+
echo $HF_HUB_TOKEN > ~/.cache/huggingface/token
231+
232+
- name: AML smoke test
233+
run: |
234+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/aio_objdet_dataset.tar.gz > /dev/null 2>&1
235+
tar -xf aio_objdet_dataset.tar.gz > /dev/null
236+
237+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 natural_language_processing/extractive_question_answering/bert_large/run_huggingface.py -m bert-large-cased-whole-word-masking-finetuned-squad
238+
239+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/resnet_50_v15_tf_fp32.pb > /dev/null 2>&1
240+
IGNORE_DATASET_LIMITS=1 AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/classification/resnet_50_v15/run.py -m resnet_50_v15_tf_fp32.pb -b 32 -p fp32 -f tf --timeout=60
39241
40242
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/ssd_inception_v2_tf_fp32.pb > /dev/null 2>&1
41-
sudo -s PYTHONPATH=./ OMP_NUM_THREADS=2 python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -p fp32 --timeout=60 --images_path=aio_objdet_dataset --anno_path=aio_objdet_dataset/annotations.json
243+
IGNORE_DATASET_LIMITS=1 python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -b 8 -p fp32 --timeout=60
244+
245+
test_onnxrt_arm64:
246+
runs-on: self-hosted
247+
container:
248+
image: amperecomputingai/onnxruntime:latest
249+
options: --memory=170g
250+
name: Ampere Altra - Ampere optimized ONNXRunTime (Docker image)
251+
env:
252+
PYTHONPATH: ./
253+
COCO_IMG_PATH: aio_objdet_dataset
254+
COCO_ANNO_PATH: aio_objdet_dataset/annotations.json
255+
AIO_NUM_THREADS: 32
256+
steps:
257+
- name: Git checkout & pull submodules
258+
uses: actions/checkout@v4
259+
with:
260+
submodules: true
261+
262+
- name: Set up AML
263+
run:
264+
bash setup_deb.sh
265+
266+
- name: AML smoke test
267+
run: |
268+
wget https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/aio_objdet_dataset.tar.gz > /dev/null 2>&1
269+
tar -xvf aio_objdet_dataset.tar.gz > /dev/null
270+
271+
wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1
272+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort
273+
274+
wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1
275+
tar -xf vgg16.tar.gz > /dev/null
276+
AIO_IMPLICIT_FP16_TRANSFORM_FILTER=".*" python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort

0 commit comments

Comments
 (0)