Skip to content

Commit 248eaf0

Browse files
snnnJaswanth51
snnn
authored and
Jaswanth51
committed
Add error handling to extract_nuget_files.ps1 (microsoft#25866)
### Description 1. Check process exit code when running 7z.exe . Currently the errors were silently ignored. 2. Add snld20 flag to the 7z.exe commands, which is needed to be compatible with the latest 7z release.
1 parent f64f5cd commit 248eaf0

File tree

2 files changed

+141
-93
lines changed

2 files changed

+141
-93
lines changed

‎tools/ci_build/github/windows/extract_nuget_files.ps1‎

Lines changed: 81 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,105 +1,119 @@
11
# Copyright (c) Microsoft Corporation. All rights reserved.
22
# Licensed under the MIT License.
33

4-
# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline
4+
# This file is used by Zip-Nuget-Java Packaging Pipeline
55

6-
# Re-construct a build directory that contains binaries from all the different platforms we're including
7-
# in the native ORT nuget package
6+
# Define the directory for NuGet artifacts.
87
$nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts"
9-
New-Item -Path $nuget_artifacts_dir -ItemType directory
8+
# Create the directory if it doesn't exist.
9+
New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue
1010

1111
## .zip files
12-
# unzip directly
13-
# exclude the iOS xcframework as we need to leave that zipped up to preserve symlinks
14-
Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\* -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip |
12+
# Unzip files directly, excluding the iOS xcframework to preserve its symlinks.
13+
Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact\*" -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip |
1514
Foreach-Object {
16-
$cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir"
17-
Write-Output $cmd
18-
Invoke-Expression -Command $cmd
15+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
16+
$arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20"
17+
Write-Output "Executing: 7z.exe $arguments"
18+
# Directly call 7z.exe using the call operator '&'
19+
& 7z.exe $arguments
20+
# Check the exit code of the last command. A non-zero code indicates an error.
21+
if ($LASTEXITCODE -ne 0) {
22+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
23+
}
1924
}
2025

2126
## .tgz files
22-
# first extract the tar file from the tgz
23-
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz |
27+
# First, extract the .tar file from the .tgz archive.
28+
Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz |
2429
Foreach-Object {
25-
$cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact"
26-
Write-Output $cmd
27-
Invoke-Expression -Command $cmd
30+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
31+
$arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20"
32+
Write-Output "Executing: 7z.exe $arguments"
33+
& 7z.exe $arguments
34+
if ($LASTEXITCODE -ne 0) {
35+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
36+
}
2837
}
2938

30-
# now extract the actual folder structure from the tar file to the build dir
31-
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar |
39+
# Now, extract the contents from the .tar file into the final directory.
40+
Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar |
3241
Foreach-Object {
33-
$cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir"
34-
Write-Output $cmd
35-
Invoke-Expression -Command $cmd
42+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
43+
$arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20"
44+
Write-Output "Executing: 7z.exe $arguments"
45+
& 7z.exe $arguments
46+
if ($LASTEXITCODE -ne 0) {
47+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
48+
}
3649
}
3750

38-
# process iOS xcframework
39-
$xcframeworks = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter onnxruntime_ios_xcframework.*.zip
51+
# Process iOS xcframework
52+
$xcframeworks = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter onnxruntime_ios_xcframework.*.zip
4053
if ($xcframeworks.Count -eq 1) {
41-
$xcframework = $xcframeworks[0]
42-
$target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework"
43-
# remove version info from filename and use required filename format
44-
$target_file = "$target_dir\onnxruntime.xcframework.zip"
45-
New-Item -Path $target_dir -ItemType directory
54+
$xcframework = $xcframeworks[0]
55+
$target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework"
56+
# Use the required filename format, removing version info.
57+
$target_file = "$target_dir\onnxruntime.xcframework.zip"
58+
New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue
4659

47-
Write-Output "Copy-Item $($xcframework.FullName) $target_file"
48-
Copy-Item $xcframework.FullName $target_file
60+
Write-Output "Copying $($xcframework.FullName) to $target_file"
61+
Copy-Item $xcframework.FullName $target_file
4962
}
5063
elseif ($xcframeworks.Count -gt 1) {
51-
Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]"
64+
Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]"
5265
}
5366

54-
55-
# copy android AAR.
56-
# for full build of onnxruntime Android AAR, there should only be one .aar file
57-
# called onnxruntime-android-x.y.z.aar or onnxruntime-training-android-x.y.z.aar but sanity check that
58-
$aars = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.aar
67+
# Copy Android AAR file.
68+
# There should only be one .aar file for a full build.
69+
$aars = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.aar
5970
if ($aars.Count -eq 1) {
60-
$aar = $aars[0]
61-
$aar_prefix = "onnxruntime"
62-
if ($aar -like "onnxruntime-training*") {
63-
$aar_prefix = "onnxruntime-training"
64-
}
65-
$target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar"
66-
$target_file = "$target_dir\onnxruntime.aar" # remove '-mobile' and version info from filename
67-
New-Item -Path $target_dir -ItemType directory
71+
$aar = $aars[0]
72+
$aar_prefix = "onnxruntime"
73+
if ($aar.Name -like "onnxruntime-training*") {
74+
$aar_prefix = "onnxruntime-training"
75+
}
76+
$target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar"
77+
# Remove version info from the filename for consistency.
78+
$target_file = "$target_dir\onnxruntime.aar"
79+
New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue
6880

69-
Write-Output "Copy-Item $($aar.FullName) $target_file"
70-
Copy-Item $aar.FullName $target_file
81+
Write-Output "Copying $($aar.FullName) to $target_file"
82+
Copy-Item $aar.FullName $target_file
7183
}
7284
elseif ($aars.Count -gt 1) {
73-
Write-Error "Expected at most one Android .aar file but got: [$aars]"
85+
Write-Error "Expected at most one Android .aar file but got: [$aars]"
7486
}
7587

76-
# Check whether this is a training pipeline
77-
$is_training_pipeline = $false
78-
if (Test-Path -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*) {
79-
$is_training_pipeline = $true
80-
Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline."
88+
# Check if this is a training pipeline by looking for a specific directory.
89+
$is_training_pipeline = Test-Path -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*"
90+
if ($is_training_pipeline) {
91+
Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline."
8192
}
8293

83-
# Copy onnxruntime and protoc binaries to the binaries dir as these are required
84-
# by Microsoft.ML.OnnxRuntime.Tests.NetCoreApp
94+
# Copy onnxruntime and protoc binaries required by tests.
95+
$destinationDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo"
8596
if ($is_training_pipeline) {
86-
Copy-Item -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo
97+
Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\*" -Destination $destinationDir -Recurse
8798
}
8899
else {
89-
Copy-Item -Path $nuget_artifacts_dir\onnxruntime-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo
100+
Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-win-x64-*\lib\*" -Destination $destinationDir -Recurse
90101
}
91102

92-
"Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-*"
93-
$ort_dirs = Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-*
94-
foreach ($ort_dir in $ort_dirs)
95-
{
96-
# remove the last '-xxx' segment from the dir name. typically that's the architecture.
97-
$dirname = Split-Path -Path $ort_dir -Leaf
98-
$dirname = $dirname.SubString(0,$dirname.LastIndexOf('-'))
99-
Write-Output "Renaming $ort_dir to $dirname"
100-
Rename-Item -Path $ort_dir -NewName $nuget_artifacts_dir\$dirname
103+
# Rename directories to remove the architecture-specific suffix.
104+
Write-Output "Renaming onnxruntime directories..."
105+
Get-ChildItem -Directory -Path "$nuget_artifacts_dir\onnxruntime-*" | ForEach-Object {
106+
$dirname = $_.Name
107+
# Find the last hyphen and remove the suffix.
108+
$lastHyphenIndex = $dirname.LastIndexOf('-')
109+
if ($lastHyphenIndex -gt -1) {
110+
$newName = $dirname.Substring(0, $lastHyphenIndex)
111+
$newPath = Join-Path -Path $_.Parent.FullName -ChildPath $newName
112+
Write-Output "Renaming '$($_.FullName)' to '$newPath'"
113+
Rename-Item -Path $_.FullName -NewName $newName
114+
}
101115
}
102116

103-
# List artifacts
104-
"Post copy artifacts"
105-
Get-ChildItem -Recurse $nuget_artifacts_dir\
117+
# List the final artifacts.
118+
Write-Output "Post-copy artifacts:"
119+
Get-ChildItem -Recurse $nuget_artifacts_dir

‎tools/ci_build/github/windows/extract_nuget_files_gpu.ps1‎

Lines changed: 60 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,81 @@
22
# Licensed under the MIT License.
33

44
# This file is used by Zip-Nuget-Java Packaging Pipeline
5-
New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts -ItemType directory
5+
# Define the directory for NuGet artifacts.
6+
$nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts"
7+
# Create the directory if it doesn't exist.
8+
New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue
69

7-
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.zip |
10+
## .zip files
11+
Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.zip |
812
Foreach-Object {
9-
$cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts"
10-
Write-Output $cmd
11-
Invoke-Expression -Command $cmd
13+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
14+
$arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20"
15+
Write-Output "Executing: 7z.exe $arguments"
16+
& 7z.exe $arguments
17+
if ($LASTEXITCODE -ne 0) {
18+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
19+
}
1220
}
1321

14-
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz |
22+
## .tgz files
23+
Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz |
1524
Foreach-Object {
16-
$cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" # *.tar will be created after *.tgz is extracted
17-
Write-Output $cmd
18-
Invoke-Expression -Command $cmd
25+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
26+
# *.tar will be created after *.tgz is extracted
27+
$arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20"
28+
Write-Output "Executing: 7z.exe $arguments"
29+
& 7z.exe $arguments
30+
if ($LASTEXITCODE -ne 0) {
31+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
32+
}
1933
}
2034

21-
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar |
35+
## .tar files
36+
Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar |
2237
Foreach-Object {
23-
$cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts"
24-
Write-Output $cmd
25-
Invoke-Expression -Command $cmd
38+
# The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01).
39+
$arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20"
40+
Write-Output "Executing: 7z.exe $arguments"
41+
& 7z.exe $arguments
42+
if ($LASTEXITCODE -ne 0) {
43+
throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE"
44+
}
2645
}
2746

47+
# Create directory for protobuf build dependencies.
48+
New-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" -ItemType directory -ErrorAction SilentlyContinue
2849

29-
New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo -ItemType directory
30-
31-
Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo
50+
# Copy CUDA libraries.
51+
Copy-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\*" -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo"
3252

53+
# Install protoc via dotnet.
3354
$protocInstallDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build"
3455
dotnet new console
3556
dotnet add package Google.Protobuf.Tools --version 3.21.12 --package-directory $protocInstallDir
57+
if ($LASTEXITCODE -ne 0) {
58+
throw "Error adding Google.Protobuf.Tools package. Exit code: $LASTEXITCODE"
59+
}
60+
61+
# Find and copy the protoc executable.
3662
$protocDir = Get-ChildItem -Path $protocInstallDir -Recurse -Filter "protoc.exe" | Select-Object -ExpandProperty DirectoryName -First 1
37-
Write-Output $protocDir
38-
Copy-Item -Path $protocDir -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo
39-
40-
$ort_dirs = Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-* -Directory
41-
foreach ($ort_dir in $ort_dirs)
42-
{
43-
$dirname = Split-Path -Path $ort_dir -Leaf
44-
$dirname = $dirname.SubString(0,$dirname.LastIndexOf('-'))
45-
Write-Output "Renaming $ort_dir to $dirname"
46-
Rename-Item -Path $ort_dir -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\$dirname
63+
if ($protocDir) {
64+
Write-Output "Found protoc directory: $protocDir"
65+
Copy-Item -Path $protocDir -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo"
66+
}
67+
else {
68+
Write-Error "Could not find protoc.exe in $protocInstallDir"
4769
}
4870

71+
# Rename onnxruntime directories to a generic format.
72+
$ort_dirs = Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-*" -Directory
73+
foreach ($ort_dir in $ort_dirs) {
74+
$dirname = Split-Path -Path $ort_dir -Leaf
75+
$lastHyphenIndex = $dirname.LastIndexOf('-')
76+
if ($lastHyphenIndex -gt -1) {
77+
$newName = $dirname.Substring(0, $lastHyphenIndex)
78+
$newPath = Join-Path -Path $ort_dir.Parent.FullName -ChildPath $newName
79+
Write-Output "Renaming '$($ort_dir.FullName)' to '$newPath'"
80+
Rename-Item -Path $ort_dir.FullName -NewName $newName
81+
}
82+
}

0 commit comments

Comments
 (0)