Skip to content

Commit 2107e81

Browse files
dschaefer2dschaefer2
authored
[6.2] Allow prebuilt manifests to be updated as platforms are added. (swiftlang#8785)
One thing we're finding with snapshot toolchain builds is that not all platforms are completed at the same time. The prebuilts publishing job builds for the libraries using the latest available toolchains and then merges them together when signing. Havoc ensues when the toolchains aren't all the same version. This change adds more metadata to the per zip json files including the swiftVersion used to build the library. The signing task will then see if a manifest already exists for that version and merge in the new platforms. If it doesn't it creates a new manifest from the other metadata in the zip json files. This takes out the use of the current swiftVersion when signing since it may not have been the version that created the libraries. Cherry-pick from main swiftlang#8784
1 parent e198274 commit 2107e81

File tree

2 files changed

+152
-47
lines changed

2 files changed

+152
-47
lines changed

‎Sources/Workspace/Workspace+Prebuilts.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ extension Workspace {
6161

6262
public struct Artifact: Identifiable, Codable {
6363
public let platform: Platform
64-
public let checksum: String
64+
public var checksum: String
6565

6666
public var id: Platform { platform }
6767

‎Sources/swift-build-prebuilts/BuildPrebuilts.swift‎

Lines changed: 151 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,18 @@ import struct TSCBasic.ByteString
2424
import struct TSCBasic.SHA256
2525
import Workspace
2626

27+
// Format for the .zip.json files.
28+
struct Artifact: Codable {
29+
var platform: Workspace.PrebuiltsManifest.Platform
30+
var checksum: String
31+
var libraryName: String?
32+
var products: [String]?
33+
var includePath: [RelativePath]?
34+
var cModules: [String]? // deprecated, includePath is the way forward
35+
var swiftVersion: String?
36+
}
37+
38+
// The master list of repos and their versions
2739
struct PrebuiltRepos: Codable {
2840
let url: URL
2941
let versions: [Version]
@@ -118,12 +130,12 @@ struct BuildPrebuilts: AsyncParsableCommand {
118130
@Option(name: .customLong("cert-chain-path"), help: "Path to a certificate (DER encoded) in the chain. The certificate used for signing must be first and the root certificate last.")
119131
var certChainPathStrs: [String] = []
120132

133+
@Option(help: .hidden)
134+
var prebuiltsUrl: String = "https://download.swift.org/prebuilts"
135+
121136
@Flag(help: .hidden)
122137
var testSigning: Bool = false
123138

124-
@Flag(name: .customLong("include-path"), help: "Add includePath to manifest")
125-
var addIncludePath: Bool = false
126-
127139
func validate() throws {
128140
if sign && !testSigning {
129141
guard privateKeyPathStr != nil else {
@@ -241,8 +253,6 @@ struct BuildPrebuilts: AsyncParsableCommand {
241253
}
242254
try packageContents.write(to: packageFile.asURL, atomically: true, encoding: .utf8)
243255

244-
var newLibraries: [Workspace.PrebuiltsManifest.Library] = []
245-
246256
// Build
247257
for library in version.manifest.libraries {
248258
let cModules = libraryTargets[library.name]?.compactMap({ $0 as? ClangModule }) ?? []
@@ -303,32 +313,29 @@ struct BuildPrebuilts: AsyncParsableCommand {
303313
let contents = try ByteString(Data(contentsOf: zipFile.asURL))
304314
#endif
305315

316+
// Manifest fragment for the zip file
306317
let checksum = SHA256().hash(contents).hexadecimalRepresentation
307-
let artifact: Workspace.PrebuiltsManifest.Library.Artifact =
308-
.init(platform: platform, checksum: checksum)
318+
let artifact = Artifact(
319+
platform: platform,
320+
checksum: checksum,
321+
libraryName: library.name,
322+
products: library.products,
323+
includePath: cModules.map({ $0.includeDir.relative(to: repoDir ) }),
324+
cModules: cModules.map(\.name),
325+
swiftVersion: swiftVersion
326+
)
309327

310328
let artifactJsonFile = versionDir.appending("\(swiftVersion)-\(library.name)-\(platform).zip.json")
311329
try fileSystem.writeFileContents(artifactJsonFile, data: encoder.encode(artifact))
312330

331+
// Clean up
313332
try fileSystem.removeFileTree(libDir)
314333
try fileSystem.removeFileTree(modulesDir)
315334
try fileSystem.removeFileTree(includesDir)
316335
}
317336

318-
let newLibrary = Workspace.PrebuiltsManifest.Library(
319-
name: library.name,
320-
products: library.products,
321-
cModules: cModules.map({ $0.name }),
322-
includePath: addIncludePath ? cModules.map({ $0.includeDir.relative(to: repoDir ) }) : nil,
323-
)
324-
newLibraries.append(newLibrary)
325-
326337
try await shell("git restore .", cwd: repoDir)
327338
}
328-
329-
let manifest = Workspace.PrebuiltsManifest(libraries: newLibraries)
330-
let manifestFile = versionDir.appending("\(swiftVersion)-prebuilts.json")
331-
try fileSystem.writeFileContents(manifestFile, data: encoder.encode(manifest))
332339
}
333340
}
334341

@@ -342,31 +349,127 @@ struct BuildPrebuilts: AsyncParsableCommand {
342349
encoder.outputFormatting = .prettyPrinted
343350
let decoder = JSONDecoder()
344351

352+
let httpClient = HTTPClient()
353+
345354
guard let swiftVersion = try computeSwiftVersion() else {
346355
print("Unable to determine swift compiler version")
347-
return
356+
_exit(1)
348357
}
349358

350359
for repo in prebuiltRepos {
351360
let prebuiltDir = stageDir.appending(repo.url.lastPathComponent)
352361
for version in repo.versions {
353362
let versionDir = prebuiltDir.appending(version.tag)
354-
let prebuiltsFile = versionDir.appending("\(swiftVersion)-prebuilts.json")
355-
let manifestFile = versionDir.appending("\(swiftVersion)-manifest.json")
356-
357-
// Load generated manifest
358-
let manifestContents: Data = try fileSystem.readFileContents(prebuiltsFile)
359-
var manifest = try decoder.decode(Workspace.PrebuiltsManifest.self, from: manifestContents)
360-
manifest.libraries = try manifest.libraries.map({
361-
var library = $0
362-
library.artifacts = try fileSystem.getDirectoryContents(versionDir)
363-
.filter({ $0.hasSuffix(".zip.json")})
364-
.compactMap({
365-
let data: Data = try fileSystem.readFileContents(versionDir.appending($0))
366-
return try? decoder.decode(Workspace.PrebuiltsManifest.Library.Artifact.self, from: data)
367-
})
368-
return library
369-
})
363+
364+
// Load artifacts
365+
let artifacts = try fileSystem.getDirectoryContents(versionDir)
366+
.filter({ $0.hasSuffix(".zip.json") })
367+
.map {
368+
let data: Data = try fileSystem.readFileContents(versionDir.appending($0))
369+
var artifact = try decoder.decode(Artifact.self, from: data)
370+
if artifact.swiftVersion == nil || artifact.libraryName == nil {
371+
let regex = try Regex(#"(.+)-([^-]+)-[^-]+.zip.json"#)
372+
if let match = try regex.firstMatch(in: $0),
373+
match.count > 2,
374+
let swiftVersion = match[1].substring,
375+
let libraryName = match[2].substring
376+
{
377+
artifact.swiftVersion = .init(swiftVersion)
378+
artifact.libraryName = .init(libraryName)
379+
}
380+
}
381+
return artifact
382+
}
383+
384+
// Fetch manifests for requested swift versions
385+
let swiftVersions: Set<String> = .init(artifacts.compactMap(\.swiftVersion))
386+
var manifests: [String: Workspace.PrebuiltsManifest] = [:]
387+
for swiftVersion in swiftVersions {
388+
let manifestFile = "\(swiftVersion)-manifest.json"
389+
let destination = versionDir.appending(component: manifestFile)
390+
if fileSystem.exists(destination) {
391+
let signedManifest = try decoder.decode(
392+
path: destination,
393+
fileSystem: fileSystem,
394+
as: Workspace.SignedPrebuiltsManifest.self
395+
)
396+
manifests[swiftVersion] = signedManifest.manifest
397+
} else {
398+
let manifestURL = URL(string: prebuiltsUrl)?.appending(components: repo.url.lastPathComponent, version.tag, manifestFile)
399+
guard let manifestURL else {
400+
print("Invalid URL \(prebuiltsUrl)")
401+
_exit(1)
402+
}
403+
404+
var headers = HTTPClientHeaders()
405+
headers.add(name: "Accept", value: "application/json")
406+
var request = HTTPClient.Request.download(
407+
url: manifestURL,
408+
headers: headers,
409+
fileSystem: fileSystem,
410+
destination: destination
411+
)
412+
request.options.retryStrategy = .exponentialBackoff(
413+
maxAttempts: 3,
414+
baseDelay: .milliseconds(50)
415+
)
416+
request.options.validResponseCodes = [200]
417+
418+
do {
419+
_ = try await httpClient.execute(request) { _, _ in }
420+
} catch {
421+
manifests[swiftVersion] = .init()
422+
continue
423+
}
424+
425+
let signedManifest = try decoder.decode(
426+
path: destination,
427+
fileSystem: fileSystem,
428+
as: Workspace.SignedPrebuiltsManifest.self
429+
)
430+
431+
manifests[swiftVersion] = signedManifest.manifest
432+
}
433+
}
434+
435+
// Merge in the artifacts
436+
for artifact in artifacts {
437+
let swiftVersion = artifact.swiftVersion ?? swiftVersion
438+
var manifest = manifests[swiftVersion, default: version.manifest]
439+
let libraryName = artifact.libraryName ?? manifest.libraries[0].name
440+
var library = manifest.libraries.first(where: { $0.name == libraryName }) ?? .init(name: libraryName)
441+
var newArtifacts = library.artifacts ?? []
442+
443+
if let products = artifact.products {
444+
library.products = products
445+
}
446+
447+
if let includePath = artifact.includePath {
448+
library.includePath = includePath
449+
}
450+
451+
if let cModules = artifact.cModules {
452+
library.cModules = cModules
453+
}
454+
455+
if let index = newArtifacts.firstIndex(where: { $0.platform == artifact.platform }) {
456+
var oldArtifact = newArtifacts[index]
457+
oldArtifact.checksum = artifact.checksum
458+
newArtifacts[index] = oldArtifact
459+
} else {
460+
newArtifacts.append(.init(platform: artifact.platform, checksum: artifact.checksum))
461+
}
462+
463+
library.artifacts = newArtifacts
464+
465+
if let index = manifest.libraries.firstIndex(where: { $0.name == libraryName }) {
466+
manifest.libraries[index] = library
467+
} else {
468+
manifest.libraries.append(library)
469+
}
470+
471+
manifests[swiftVersion] = manifest
472+
}
370473

371474
if testSigning {
372475
// Use SwiftPM's test certificate chain and private key for testing
@@ -401,19 +504,21 @@ struct BuildPrebuilts: AsyncParsableCommand {
401504
observabilityScope: ObservabilitySystem { _, diagnostic in print(diagnostic) }.topScope
402505
)
403506

404-
let signature = try await signer.sign(
405-
manifest: manifest,
406-
certChainPaths: certChainPaths,
407-
certPrivateKeyPath: privateKeyPath,
408-
fileSystem: fileSystem
409-
)
410-
411-
let signedManifest = Workspace.SignedPrebuiltsManifest(manifest: manifest, signature: signature)
412-
try encoder.encode(signedManifest).write(to: manifestFile.asURL)
507+
for (swiftVersion, manifest) in manifests where !manifest.libraries.flatMap({ $0.artifacts ?? [] }).isEmpty {
508+
let signature = try await signer.sign(
509+
manifest: manifest,
510+
certChainPaths: certChainPaths,
511+
certPrivateKeyPath: privateKeyPath,
512+
fileSystem: fileSystem
513+
)
514+
515+
let signedManifest = Workspace.SignedPrebuiltsManifest(manifest: manifest, signature: signature)
516+
let manifestFile = versionDir.appending(component: "\(swiftVersion)-manifest.json")
517+
try encoder.encode(signedManifest).write(to: manifestFile.asURL)
518+
}
413519
}
414520
}
415521
}
416-
417522
}
418523

419524
func canBuild(_ platform: Workspace.PrebuiltsManifest.Platform) -> Bool {

0 commit comments

Comments
 (0)