Skip to content

Commit ffaddea

Browse files
authored
Refactor cuda packaging pipeline (microsoft#22542)
### Description <!-- Describe your changes. --> ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
1 parent 0028d3f commit ffaddea

11 files changed

+172
-261
lines changed

tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,8 @@ stages:
3434
# build Python packages
3535
# Linux GPU only
3636
- ${{ if parameters.BuildPythonPackages }}:
37-
- template: templates/py-packaging-stage.yml
37+
- template: stages/py-gpu-packaging-stage.yml
3838
parameters:
3939
enable_linux_gpu: true
40-
enable_linux_cpu: false
41-
enable_windows_cpu: false
4240
enable_windows_gpu: false
43-
enable_mac_cpu: false
44-
enable_linux_arm: false
41+
cuda_version: 12.2
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
trigger: none
2+
3+
parameters:
4+
- name: enable_linux_cuda
5+
type: boolean
6+
default: true
7+
8+
- name: enable_windows_cuda
9+
type: boolean
10+
default: true
11+
12+
- name: cmake_build_type
13+
type: string
14+
default: 'Release'
15+
values:
16+
- Debug
17+
- Release
18+
- RelWithDebInfo
19+
- MinSizeRel
20+
21+
stages:
22+
- template: stages/py-gpu-packaging-stage.yml
23+
parameters:
24+
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
25+
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
26+
cmake_build_type: ${{ parameters.cmake_build_type }}
27+
cuda_version: '11.8'
Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
trigger: none
2-
2+
# The `resources` specify the location and version of the 1ES PT.
3+
resources:
4+
repositories:
5+
- repository: 1esPipelines
6+
type: git
7+
name: 1ESPipelineTemplates/1ESPipelineTemplates
8+
ref: refs/tags/release
39
parameters:
4-
- name: enable_linux_gpu
10+
- name: enable_linux_cuda
511
type: boolean
612
default: true
7-
- name: enable_windows_gpu
13+
14+
- name: enable_windows_cuda
815
type: boolean
916
default: true
17+
1018
- name: cmake_build_type
1119
type: string
1220
default: 'Release'
@@ -15,28 +23,22 @@ parameters:
1523
- Release
1624
- RelWithDebInfo
1725
- MinSizeRel
18-
- name: cuda_version
19-
type: string
20-
default: '12.2'
21-
values:
22-
- 11.8
23-
- 12.2
24-
- name: SpecificArtifact
25-
displayName: Use Specific Artifact
26-
type: boolean
27-
default: false
2826

29-
- name: BuildId
30-
displayName: Specific Artifact's BuildId
31-
type: string
32-
default: '0'
27+
extends:
28+
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
29+
# For non-production pipelines, use "Unofficial" as defined below.
30+
# For productions pipelines, use "Official".
31+
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
32+
parameters:
33+
# Update the pool with your team's 1ES hosted pool.
34+
pool:
35+
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
36+
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
3337

34-
stages:
35-
- template: stages/py-cuda-packaging-stage.yml
36-
parameters:
37-
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
38-
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
39-
cmake_build_type: ${{ parameters.cmake_build_type }}
40-
cuda_version: ${{ parameters.cuda_version }}
41-
SpecificArtifact: ${{ parameters.SpecificArtifact }}
42-
BuildId: ${{ parameters.BuildId }}
38+
stages:
39+
- template: stages/py-gpu-packaging-stage.yml
40+
parameters:
41+
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
42+
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
43+
cmake_build_type: ${{ parameters.cmake_build_type }}
44+
cuda_version: '12.2'
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
trigger: none
2+
3+
parameters:
4+
- name: cmake_build_type
5+
type: string
6+
default: 'Release'
7+
values:
8+
- Debug
9+
- Release
10+
- RelWithDebInfo
11+
- MinSizeRel
12+
13+
stages:
14+
- template: stages/py-gpu-packaging-stage.yml
15+
parameters:
16+
enable_windows_dml: true
17+
cmake_build_type: ${{ parameters.cmake_build_type }}
18+
publish_symbols: true

tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,11 @@ parameters:
44
type: boolean
55
default: true
66

7-
- name: enable_linux_gpu
8-
displayName: 'Whether Linux GPU package is built.'
9-
type: boolean
10-
default: true
11-
127
- name: enable_windows_cpu
138
displayName: 'Whether Windows CPU package is built.'
149
type: boolean
1510
default: true
1611

17-
- name: enable_windows_gpu
18-
displayName: 'Whether Windows GPU package is built.'
19-
type: boolean
20-
default: true
21-
2212
- name: enable_mac_cpu
2313
displayName: 'Whether Mac CPU package is built.'
2414
type: boolean
@@ -74,12 +64,10 @@ parameters:
7464
trigger: none
7565

7666
stages:
77-
- template: templates/py-packaging-stage.yml
67+
- template: stages/py-cpu-packaging-stage.yml
7868
parameters:
79-
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
8069
enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
8170
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
82-
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
8371
enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
8472
enable_linux_arm: ${{ parameters.enable_linux_arm }}
8573
enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
@@ -90,3 +78,4 @@ stages:
9078
cmake_build_type: ${{ parameters.cmake_build_type }}
9179
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
9280
publish_symbols: true
81+

tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,15 @@ jobs:
5757
- checkout: self
5858
- task: DownloadPipelineArtifact@2
5959
inputs:
60-
artifact: 'drop-linux-gpu-x86_64'
61-
targetPath: '$(Build.SourcesDirectory)/drop-linux-gpu-x86_64'
60+
artifact: 'linux_gpu_wheel_x86_64'
61+
targetPath: '$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64'
6262
${{ if ne(parameters.build_id, 'latest') }}:
6363
buildType: 'specific'
6464
project: '${{ parameters.project }}'
6565
pipeline: '${{ parameters.pipeline }}'
6666
buildVersionToDownload: 'specific'
6767
buildId: '${{ parameters.build_id }}'
68-
displayName: 'Download Build Artifacts - drop-linux-gpu-x86_64'
68+
displayName: 'Download Build Artifacts - linux_gpu_wheel_x86_64'
6969

7070
- task: DownloadPipelineArtifact@2
7171
inputs:
@@ -82,7 +82,7 @@ jobs:
8282
- bash: |
8383
set -e -x
8484
ls $(Build.SourcesDirectory)
85-
mv "$(Build.SourcesDirectory)/drop-linux-gpu-x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}}
85+
mv "$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}}
8686
mv "$(Build.SourcesDirectory)/onnxruntime_gpu" "$(Build.BinariesDirectory)/whl"
8787
cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp
8888
find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \;

0 commit comments

Comments
 (0)