Refactor all packing to Pack-Crates.ps1

Author: danieljurek

eng/pipelines/templates/jobs/pack.yml

@@ -69,31 +69,37 @@ jobs:
   - ${{ else }}:
     - pwsh: |
         $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
-        $requireDependencies = $true
+        $isReleaseBuild = $true
         $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' }
 
         if (!$artifactsToBuild) {
           Write-Host "No packages to release. Building all packages in the service directory with no dependency validation."
           $artifactsToBuild = $artifacts
-          $requireDependencies = $false
+          $isReleaseBuild = $false
         }
 
         $packageNames = $artifactsToBuild.name
 
-        if ($requireDependencies) {
-          Write-Host "Building crates with dependency validation for release"
-          & $(Build.SourcesDirectory)/eng/scripts/Pack-ReleaseCrates.ps1 `
-            -OutputPath '$(Build.ArtifactStagingDirectory)' `
-            -PackageNames $packageNames `
-            -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json'
+        Write-Host "##vso[task.setvariable variable=PackageNames]$($packageNames -join ',')"
+        if ($isReleaseBuild) {
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]-Release"
         } else {
-          Write-Host "Building crates without validating dependencies"
-          & $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1 `
-            -OutputPath '$(Build.ArtifactStagingDirectory)' `
-            -PackageNames $packageNames
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]"
         }
+      displayName: Configure crate packing
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+
+    - task: Powershell@2
       displayName: Pack Crates
       condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
+        arguments: >
+          -OutputPath '$(Build.ArtifactStagingDirectory)'
+          -PackageNames $(PackageNames)
+          $(AdditionalPackageParams)
+
 
   - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
     parameters:

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -95,6 +95,12 @@ stages:
                         # and use ArtifactIndex to select the right one
                         $index = [int]'$(ArtifactIndex)'
                         $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json
+                        
+                        # Force $artifacts to be an array (PowerShell unrolls single-item arrays)
+                        if ($artifacts -isnot [Array]) {                          
+                          $artifacts = @($artifacts)
+                        }
+
                         if ($index -ge $artifacts.Count) {
                           Write-Error "ArtifactIndex $index is out of range (0..$($artifacts.Count - 1))"
                           exit 1

eng/scripts/Pack-Common.ps1

@@ -8,13 +8,14 @@ param(
   [string[]]$PackageNames,
   [Parameter(ParameterSetName = 'PackageInfo')]
   [string]$PackageInfoDirectory,
-  [switch]$NoVerify
+  [switch]$Release,
+  [switch]$NoVerify,
+  [string]$OutBuildOrderFile
 )
 
 $ErrorActionPreference = 'Stop'
 
 . ([System.IO.Path]::Combine($PSScriptRoot, '..', 'common', 'scripts', 'common.ps1'))
-. ([System.IO.Path]::Combine($PSScriptRoot, 'Pack-Common.ps1'))
 
 Write-Host @"
 Packing crates with
@@ -61,8 +62,8 @@ function Get-OutputPackageNames($workspacePackages) {
 function Get-CargoPackages() {
   $metadata = Get-CargoMetadata
 
-  # path based depdenencies are assumed to be unreleased package versions
-  # they must be included in this build and build before packages that depend on them
+  # Path based dependencies are assumed to be unreleased package versions. In
+  # non-release builds these should be packed as well.
   foreach ($package in $metadata.packages) {
     $package.UnreleasedDependencies = @()
     foreach ($dependency in $package.dependencies) {
@@ -80,9 +81,13 @@ function Get-PackagesToBuild() {
   $packages = Get-CargoPackages
   $outputPackageNames = Get-OutputPackageNames $packages
 
-  # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built
   [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) }
 
+  if ($Release) { 
+    return $packagesToBuild
+  }
+
+  # If not releasing, expand dependencies into list of packages to build
   $toProcess = $packagesToBuild
   while ($toProcess.Length -gt 0) {
     $package = $toProcess[0]
@@ -96,111 +101,77 @@ function Get-PackagesToBuild() {
     }
   }
 
-  $buildOrder = @()
-
-  # Then we order the packages to that dependencies are built first
-  while ($packagesToBuild.Count -gt 0) {
-    # Pick any package with no unreleased dependencies, add it to the build order and remove it from the list of other packages' unreleased dependencies
-    $package = $packagesToBuild | Where-Object { $_.UnreleasedDependencies.Count -eq 0 } | Select-Object -First 1
-
-    if (-not $package) {
-      Write-Error "These packages cannot be built because they depend on unreleased dependencies that aren't being built." -ErrorAction Continue
-      foreach ($package in $packagesToBuild) {
-        Write-Error "  $($package.name) -> $($package.UnreleasedDependencies -join ', ')" -ErrorAction Continue
-      }
-      exit 1
-    }
-
-    $package.OutputPackage = $outputPackageNames.Contains($package.name)
-    $buildOrder += $package
-    $packagesToBuild = @($packagesToBuild -ne $package)
-
-    foreach ($otherPackage in $packagesToBuild) {
-      $otherPackage.UnreleasedDependencies = $otherPackage.UnreleasedDependencies -ne $package
-    }
-  }
-
-  return $buildOrder
+  return $packagesToBuild
 }
 
-function Initialize-VendorDirectory() {
-  $path = "$RepoRoot/target/vendor"
-  Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host
-  return $path
+function Get-CargoMetadata() {
+  cargo metadata --no-deps --format-version 1 --manifest-path "$RepoRoot/Cargo.toml" | ConvertFrom-Json -Depth 100 -AsHashtable
 }
 
-function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) {
-  $packageName = $Package.name
-  $packageVersion = $Package.version
-
-  # create an index entry for the package
-  $packagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
+function Create-ApiViewFile($package) {
+  $packageName = $package.name
+  $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName"
+  Invoke-LoggedCommand $command -GroupOutput | Out-Host
 
-  Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'"
-  Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse
+  $packagePath = Split-Path -Path $package.manifest_path -Parent
 
-  #write an empty checksum file
-  '{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8
+  "$packagePath/review/$packageName.rust.json"
 }
 
-Push-Location $RepoRoot
+$originalLocation = Get-Location
 try {
-  $localRegistryPath = Initialize-VendorDirectory
+  Set-Location $RepoRoot
 
   [array]$packages = Get-PackagesToBuild
-
-  Write-Host "Building packages in the following order:"
+  $packageParams = @()
   foreach ($package in $packages) {
-    $packageName = $package.name
-    $type = if ($package.OutputPackage) { "output" } else { "dependency" }
-    Write-Host "  $packageName ($type)"
+    $packageParams += "--package", $package.name
   }
 
-  foreach ($package in $packages) {
-    Write-Host ""
+  if ($NoVerify) {
+    $packageParams += "--no-verify"
+  }
 
-    $packageName = $package.name
-    $packageVersion = $package.version
+  Write-Host "> cargo publish --locked --dry-run --allow-dirty $($packageParams -join ' ')"
+  & cargo publish --locked --dry-run --allow-dirty @packageParams 2>&1 | Tee-Object -Variable packResult
+  if ($LASTEXITCODE) {
+    Write-Host "cargo publish failed with exit code $LASTEXITCODE"
+    exit $LASTEXITCODE
+  }
 
-    $command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
+  if ($OutputPath -and $package.OutputPackage) {
+    $sourcePath = [System.IO.Path]::Combine($RepoRoot, "target", "package", "$packageName-$packageVersion")
+    $targetPath = [System.IO.Path]::Combine($OutputPath, $packageName)
+    $targetContentsPath = [System.IO.Path]::Combine($targetPath, "contents")
+    $targetApiReviewFile = [System.IO.Path]::Combine($targetPath, "$packageName.rust.json")
 
-    if ($NoVerify) {
-      $command += " --no-verify"
+    if (Test-Path -Path $targetContentsPath) {
+      Remove-Item -Path $targetContentsPath -Recurse -Force
     }
 
-    Invoke-LoggedCommand -Command $command -GroupOutput
+    Write-Host "Copying package '$packageName' to '$targetContentsPath'"
+    New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null
+    Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig"
 
+    Write-Host "Creating API review file"
+    $apiReviewFile = Create-ApiViewFile $package
+      
+    Write-Host "Copying API review file to '$targetApiReviewFile'"
+    Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force
+  }
 
-    # copy the package to the local registry
-    Add-CrateToLocalRegistry `
-      -LocalRegistryPath $localRegistryPath `
-      -Package $package
-
-    if ($OutputPath -and $package.OutputPackage) {
-      $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion"
-      $targetPath = "$OutputPath/$packageName"
-      $targetContentsPath = "$targetPath/contents"
-      $targetApiReviewFile = "$targetPath/$packageName.rust.json"
-
-      if (Test-Path -Path $targetContentsPath) {
-        Remove-Item -Path $targetContentsPath -Recurse -Force
+  if ($OutBuildOrderFile) {
+    $buildOrder = @()
+    foreach ($line in $packResult) { 
+      if ($line -match '^\s*Packaging (\w*) ([\w\d\.-]*)') {
+        $buildOrder += $matches[1]
       }
-
-      Write-Host "Copying package '$packageName' to '$targetContentsPath'"
-      New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null
-      Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig"
-
-      Write-Host "Creating API review file"
-      $apiReviewFile = Create-ApiViewFile $package
-      
-      Write-Host "Copying API review file to '$targetApiReviewFile'"
-      Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force
     }
-  }
 
-  Write-Host "Removing local registry"
-  Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null
+    Write-Host "Build Order: $($buildOrder -join ', ')"
+    $buildOrder | ConvertTo-Json -Depth 100 | Set-Content $OutBuildOrderFile
+  }
 }
 finally {
-  Pop-Location
+  Set-Location $originalLocation
 }