Merge pull request #1039 from nkcsgexi/additional-framework-search-path

PrebuiltModuleGen: add additional framework search paths

Author: nkcsgexi

Sources/SwiftDriver/Driver/Driver.swift

@@ -349,13 +349,20 @@ public struct Driver {
     return VirtualPath.lookup(rawSdkPath)
   } ()
 
-  lazy var iosMacFrameworksSearchPath: VirtualPath = {
+  lazy var iosMacFrameworksBase: VirtualPath = {
     sdkPath!
       .appending(component: "System")
       .appending(component: "iOSSupport")
       .appending(component: "System")
       .appending(component: "Library")
-      .appending(component: "Frameworks")
+  } ()
+
+  lazy var iosMacFrameworksSearchPath: VirtualPath = {
+    iosMacFrameworksBase.appending(component: "Frameworks")
+  } ()
+
+  lazy var iosMacPrivateFrameworksSearchPath: VirtualPath = {
+    iosMacFrameworksBase.appending(component: "PrivateFrameworks")
   } ()
 
   lazy var abiDescriptorPath: TypedVirtualPath? = {

Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 import TSCBasic
 import SwiftOptions
+import Foundation
 
 @_spi(Testing) public func isIosMacInterface(_ path: VirtualPath) throws -> Bool {
   let data = try localFileSystem.readFileContents(path).cString
@@ -241,14 +242,54 @@ public struct PrebuiltModuleInput {
   }
 }
 
+public class SwiftAdopter: Codable {
+  public let name: String
+  public let moduleDir: String
+  public let hasInterface: Bool
+  public let hasModule: Bool
+  public let isFramework: Bool
+  public let isPrivate: Bool
+  init(_ name: String, _ moduleDir: AbsolutePath, _ hasInterface: AbsolutePath?, _ hasModule: AbsolutePath?) {
+    self.name = name
+    self.moduleDir = SwiftAdopter.relativeToSDK(moduleDir)
+    self.hasInterface = hasInterface != nil
+    self.hasModule = hasModule != nil
+    self.isFramework = self.moduleDir.contains("\(name).framework")
+    self.isPrivate = self.moduleDir.contains("PrivateFrameworks")
+  }
+  static func relativeToSDK(_ fullPath: AbsolutePath) -> String {
+    var SDKDir: AbsolutePath = fullPath
+    while(SDKDir.extension != "sdk") {
+      SDKDir = SDKDir.parentDirectory
+    }
+    assert(SDKDir.extension == "sdk")
+    SDKDir = SDKDir.parentDirectory
+    return fullPath.relative(to: SDKDir).pathString
+  }
+
+  static public func emitSummary(_ adopters: [SwiftAdopter], to logDir: AbsolutePath?) throws {
+    guard let logDir = logDir else { return }
+    if !localFileSystem.exists(logDir) {
+      try localFileSystem.createDirectory(logDir, recursive: true)
+    }
+    let data = try JSONEncoder().encode(adopters)
+    if let json = try? JSONSerialization.jsonObject(with: data, options: .mutableContainers),
+       let jsonData = try? JSONSerialization.data(withJSONObject: json, options: .prettyPrinted) {
+      try localFileSystem.writeFileContents(logDir.appending(component: "adopters.json"), bytes: ByteString(jsonData))
+    }
+  }
+}
+
 typealias PrebuiltModuleOutput = PrebuiltModuleInput
 
 public struct SDKPrebuiltModuleInputsCollector {
   let sdkPath: AbsolutePath
   let nonFrameworkDirs = [RelativePath("usr/lib/swift"),
                           RelativePath("System/iOSSupport/usr/lib/swift")]
   let frameworkDirs = [RelativePath("System/Library/Frameworks"),
-                      RelativePath("System/iOSSupport/System/Library/Frameworks")]
+                       RelativePath("System/Library/PrivateFrameworks"),
+                       RelativePath("System/iOSSupport/System/Library/Frameworks"),
+                       RelativePath("System/iOSSupport/System/Library/PrivateFrameworks")]
   let sdkInfo: DarwinToolchain.DarwinSDKInfo
   let diagEngine: DiagnosticsEngine
   public init(_ sdkPath: AbsolutePath, _ diagEngine: DiagnosticsEngine) {
@@ -296,7 +337,8 @@ public struct SDKPrebuiltModuleInputsCollector {
     }
   }
 
-  public func collectSwiftInterfaceMap() throws -> [String: [PrebuiltModuleInput]] {
+  public func collectSwiftInterfaceMap() throws -> (inputMap: [String: [PrebuiltModuleInput]], adopters: [SwiftAdopter]) {
+    var allSwiftAdopters: [SwiftAdopter] = []
     var results: [String: [PrebuiltModuleInput]] = [:]
 
     func updateResults(_ dir: AbsolutePath) throws {
@@ -307,7 +349,8 @@ public struct SDKPrebuiltModuleInputsCollector {
       if results[moduleName] == nil {
         results[moduleName] = []
       }
-
+      var hasInterface: AbsolutePath?
+      var hasModule: AbsolutePath?
       // Search inside a .swiftmodule directory for any .swiftinterface file, and
       // add the files into the dictionary.
       // Duplicate entries are discarded, otherwise llbuild will complain.
@@ -320,11 +363,14 @@ public struct SDKPrebuiltModuleInputsCollector {
           if !results[moduleName]!.contains(where: { $0.path.file.basenameWithoutExt == currentBaseName }) {
             results[moduleName]!.append(PrebuiltModuleInput(interfacePath))
           }
+          hasInterface = currentFile
         }
         if currentFile.extension == "swiftmodule" {
           diagEngine.emit(warning: "found \(currentFile)")
+          hasModule = currentFile
         }
       }
+      allSwiftAdopters.append(SwiftAdopter(moduleName, dir, hasInterface, hasModule))
     }
     // Search inside framework dirs in an SDK to find .swiftmodule directories.
     for dir in frameworkDirs {
@@ -356,7 +402,7 @@ public struct SDKPrebuiltModuleInputsCollector {
         }
       }
     }
-    return sanitizeInterfaceMap(results)
+    return (inputMap: sanitizeInterfaceMap(results), adopters: allSwiftAdopters)
   }
 }
 
@@ -448,10 +494,12 @@ extension Driver {
                                                         _ dependencies: [TypedVirtualPath], _ currentABIDir: AbsolutePath?,
                                                         _ baselineABIDir: AbsolutePath?) throws -> [Job] {
     assert(inputPath.path.file.basenameWithoutExt == outputPath.path.file.basenameWithoutExt)
+    let sdkPath = sdkPath!
+    let isInternal = sdkPath.basename.hasSuffix(".Internal.sdk")
     var commandLine: [Job.ArgTemplate] = []
     commandLine.appendFlag(.compileModuleFromInterface)
     commandLine.appendFlag(.sdk)
-    commandLine.append(.path(sdkPath!))
+    commandLine.append(.path(sdkPath))
     commandLine.appendFlag(.prebuiltModuleCachePath)
     commandLine.appendPath(prebuiltModuleDir)
     commandLine.appendFlag(.moduleName)
@@ -467,6 +515,16 @@ extension Driver {
     if try isIosMacInterface(inputPath.path.file) {
       commandLine.appendFlag(.Fsystem)
       commandLine.append(.path(iosMacFrameworksSearchPath))
+      if isInternal {
+        commandLine.appendFlag(.Fsystem)
+        commandLine.append(.path(iosMacPrivateFrameworksSearchPath))
+      }
+    }
+    if isInternal {
+      commandLine.appendFlag(.Fsystem)
+      commandLine.append(.path(sdkPath.appending(component: "System")
+        .appending(component: "Library")
+        .appending(component: "PrivateFrameworks")))
     }
     // Use the specified module cache dir
     if let mcp = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle {

Sources/swift-build-sdk-interfaces/main.swift

@@ -67,6 +67,7 @@ do {
     diagnosticsEngine.emit(error: "cannot find sdk: \(sdkPath.pathString)")
     exit(1)
   }
+  let logDir = try getArgumentAsPath("-log-path")
   let collector = SDKPrebuiltModuleInputsCollector(sdkPath, diagnosticsEngine)
   var outputDir = try VirtualPath(path: rawOutputDir).absolutePath!
   // if the given output dir ends with 'prebuilt-modules', we should
@@ -108,7 +109,11 @@ do {
                               .appending(component: "SystemVersion.plist"),
                            to: sysVersionFile)
   let processSet = ProcessSet()
-  let inputMap = try collector.collectSwiftInterfaceMap()
+  let inputTuple = try collector.collectSwiftInterfaceMap()
+  let allAdopters = inputTuple.adopters
+  let currentABIDir = try getArgumentAsPath("-current-abi-dir")
+  try SwiftAdopter.emitSummary(allAdopters, to: currentABIDir)
+  let inputMap = inputTuple.inputMap
   let allModules = coreMode ? ["Foundation"] : Array(inputMap.keys)
   try withTemporaryFile(suffix: ".swift") {
     let tempPath = $0.path
@@ -131,8 +136,6 @@ do {
       args.append(mcpFlag)
       args.append(mcp)
     }
-    let logDir = try getArgumentAsPath("-log-path")
-    let currentABIDir = try getArgumentAsPath("-current-abi-dir")
     let baselineABIDir = try getArgumentAsPath("-baseline-abi-dir")
     var driver = try Driver(args: args,
                             diagnosticsEngine: diagnosticsEngine,

TestInputs/mock-sdk.Internal.sdk/SDKSettings.json

@@ -0,0 +1,8 @@
+{
+	"Version": "10.15",
+	"VersionMap": {
+		"macOS_iOSMac": {},
+		"iOSMac_macOS": {}
+	},
+	"CanonicalName": "macosx10.15"
+}

TestInputs/mock-sdk.Internal.sdk/usr/lib/swift/A.swiftmodule/arm64-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A
+import Swift
+public func FuncA() { }

TestInputs/mock-sdk.Internal.sdk/usr/lib/swift/A.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A
+import Swift
+public func FuncA() { }

TestInputs/mock-sdk.Internal.sdk/usr/lib/swift/A.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A
+import Swift
+public func FuncA() { }

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -1310,7 +1310,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
         testInputsPath.appending(component: "mock-sdk.sdk").pathString
     let diagnosticEnging = DiagnosticsEngine()
     let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPath).absolutePath!, diagnosticEnging)
-    let interfaceMap = try collector.collectSwiftInterfaceMap()
+    let interfaceMap = try collector.collectSwiftInterfaceMap().inputMap
 
     // Check interface map always contain everything
     XCTAssertTrue(interfaceMap["Swift"]!.count == 3)
@@ -1486,7 +1486,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     let baselineABIPath: String =
         testInputsPath.appending(component: "ABIBaselines").pathString
     let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPath).absolutePath!, DiagnosticsEngine())
-    let interfaceMap = try collector.collectSwiftInterfaceMap()
+    let interfaceMap = try collector.collectSwiftInterfaceMap().inputMap
     try withTemporaryDirectory { path in
       let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
       try localFileSystem.writeFileContents(main) {
@@ -1510,5 +1510,43 @@ final class ExplicitModuleBuildTests: XCTestCase {
       try abiCheckJobs.forEach { try checkABICheckingJob($0) }
     }
   }
+  func testPrebuiltModuleInternalSDK() throws {
+    let mockSDKPath = testInputsPath.appending(component: "mock-sdk.Internal.sdk")
+    let mockSDKPathStr: String = mockSDKPath.pathString
+    let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPathStr).absolutePath!, DiagnosticsEngine())
+    let interfaceMap = try collector.collectSwiftInterfaceMap().inputMap
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import A\n"
+      }
+      let moduleCachePath = "/tmp/module-cache"
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPathStr,
+                                     "-module-cache-path", moduleCachePath
+                                    ])
+      let (jobs, _) = try driver.generatePrebuitModuleGenerationJobs(with: interfaceMap,
+                                                                     into: path,
+                                                                     exhaustive: true)
+      let compileJobs = jobs.filter {$0.kind == .compile}
+      XCTAssertTrue(!compileJobs.isEmpty)
+      XCTAssertTrue(compileJobs.allSatisfy { $0.commandLine.contains(.flag("-suppress-warnings")) })
+      let PFPath = mockSDKPath.appending(component: "System").appending(component: "Library")
+        .appending(component: "PrivateFrameworks")
+      XCTAssertTrue(compileJobs.allSatisfy { $0.commandLine.contains(.path(VirtualPath.absolute(PFPath))) })
+    }
+  }
+  func testCollectSwiftAdopters() throws {
+    let mockSDKPath = testInputsPath.appending(component: "mock-sdk.Internal.sdk")
+    let mockSDKPathStr: String = mockSDKPath.pathString
+    let collector = try SDKPrebuiltModuleInputsCollector(VirtualPath(path: mockSDKPathStr).absolutePath!, DiagnosticsEngine())
+    let adopters = try collector.collectSwiftInterfaceMap().adopters
+    XCTAssertTrue(!adopters.isEmpty)
+    let A = adopters.first {$0.name == "A"}!
+    XCTAssertFalse(A.isFramework)
+    XCTAssertFalse(A.isPrivate)
+    XCTAssertFalse(A.hasModule)
+    XCTAssertTrue(A.hasInterface)
+  }
 #endif
 }