[Runtime] Harden SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH.

Have SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH use dlopen with RTLD_NOLOAD. We don't want it to allow loading arbitrary dylibs. Instead, a user can use something like DYLD_INSERT_LIBRARIES to load the dylib, which we will then pick up in the runtime, and processes that deny DYLD_INSERT_LIBRARIES will not be able to work around it with SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH.

rdar://123643585

Author: mikeash

stdlib/public/runtime/EnvironmentVariables.def

@@ -92,7 +92,9 @@ VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED, bool, true,
          "Enable use of prespecializations library.")
 
 VARIABLE(SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH, string, "",
-         "A path to a prespecializations library to use at runtime.")
+         "A path to a prespecializations library to use at runtime. In order to"
+         "be used, this library must be loaded into the process by other means"
+         "(such as DYLD_INSERT_LIBRARIES) before Swift tries to use it.")
 
 VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_LOGGING, bool, false,
          "Enable debug logging of prespecializations library use.")

stdlib/public/runtime/LibPrespecialized.cpp

@@ -35,7 +35,10 @@ static const LibPrespecializedData<InProcess> *findLibPrespecialized() {
 #if USE_DLOPEN
   auto path = runtime::environment::SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH();
   if (path && path[0]) {
-    void *handle = dlopen(path, RTLD_LAZY);
+    // Use RTLD_NOLOAD to avoid actually loading the library. We just want to
+    // find it if it has already been loaded by other means, such as
+    // DYLD_INSERT_LIBRARIES.
+    void *handle = dlopen(path, RTLD_LAZY | RTLD_NOLOAD);
     if (!handle) {
       swift::warning(0, "Failed to load prespecializations library: %s\n",
                      dlerror());

test/ExternalGenericMetadataBuilder/Inputs/testMetadataLibrary.swift

@@ -0,0 +1,26 @@
+public struct GenericStruct<T, U, V> {
+  var t: T
+  var u: U
+  var v: V
+  var str: String
+}
+
+public struct GenericField<T, U> {
+  var field: GenericStruct<T, U, Double>
+  var int: Int
+}
+
+public struct Box<T> {
+  var field: T
+}
+
+public struct Box3<T, U, V> {
+  var field1: T
+  var field2: U
+  var field3: V
+}
+
+// The protocol conformance puts a symbol into __DATA_CONST which the builder
+// can use as the base symbol for references to other data.
+public protocol PublicProto {}
+extension Box3: PublicProto {}

test/ExternalGenericMetadataBuilder/VerifyExternalMetadata.swift

@@ -1,5 +1,9 @@
 // RUN: %empty-directory(%t)
-// RUN: %target-build-swift -Xfrontend -disable-availability-checking -I %swift-lib-dir -I %swift_src_root/lib/ExternalGenericMetadataBuilder -enable-experimental-feature Extern %s -o %t/VerifyExternalMetadata
+//
+// RUN: %host-build-swift %S/Inputs/testMetadataLibrary.swift -emit-library -emit-module -o %t/libtestMetadataLibrary.dylib
+// RUN: %target-codesign %t/libtestMetadataLibrary.dylib
+//
+// RUN: %target-build-swift -Xfrontend -disable-availability-checking -I %swift-lib-dir -I %swift_src_root/lib/ExternalGenericMetadataBuilder -I %t -L %t -ltestMetadataLibrary -enable-experimental-feature Extern %s -o %t/VerifyExternalMetadata
 // RUN: %target-codesign %t/VerifyExternalMetadata
 //
 // RUN: %host-build-swift -Xfrontend -disable-availability-checking -I %swift-lib-dir -I %swift_src_root/lib/ExternalGenericMetadataBuilder -L%swift-lib-dir -lswiftGenericMetadataBuilder -Xlinker -rpath -Xlinker %swift-lib-dir -enable-experimental-feature Extern %S/Inputs/buildMetadataJSON.swift -o %t/buildMetadataJSON
@@ -11,12 +15,12 @@
 // RUN: %target-run %t/VerifyExternalMetadata getJSON > %t/names.json
 // RUN: %target-run %t/buildMetadataJSON %target-arch %t/VerifyExternalMetadata %stdlib_dir/libswiftCore.dylib < %t/names.json > %t/libswiftPrespecialized.json
 // RUN: %target-run %t/json2c %t/libswiftPrespecialized.json > %t/libswiftPrespecialized.c
-// RUN: %clang -isysroot %sdk -target %target-triple -bundle %t/libswiftPrespecialized.c -L%stdlib_dir -lswiftCore -bundle_loader %t/VerifyExternalMetadata -o %t/libswiftPrespecialized.bundle
+// RUN: %clang -isysroot %sdk -target %target-triple -dynamiclib %t/libswiftPrespecialized.c -L%stdlib_dir -lswiftCore -o %t/libswiftPrespecialized.dylib
 //
 // Set a custom library path because we need to ensure we don't load an arm64e
 // dylib into an arm64 test, since the prespecialized metadata depends on the
 // exact contents of the library.
-// RUN: env SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_LOGGING=y SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH=%t/libswiftPrespecialized.bundle %target-run env DYLD_LIBRARY_PATH=%stdlib_dir/%target-arch %t/VerifyExternalMetadata
+// RUN: env SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_LOGGING=y DYLD_INSERT_LIBRARIES=%t/libswiftPrespecialized.dylib SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH=%t/libswiftPrespecialized.dylib %target-run env DYLD_LIBRARY_PATH=%stdlib_dir/%target-arch %t/VerifyExternalMetadata
 
 // REQUIRES: executable_test
 // REQUIRES: OS=macosx && CPU=arm64
@@ -25,32 +29,7 @@ import ExternalGenericMetadataBuilder
 import Foundation
 import StdlibUnittest
 
-public struct GenericStruct<T, U, V> {
-  var t: T
-  var u: U
-  var v: V
-  var str: String
-}
-
-public struct GenericField<T, U> {
-  var field: GenericStruct<T, U, Double>
-  var int: Int
-}
-
-public struct Box<T> {
-  var field: T
-}
-
-public struct Box3<T, U, V> {
-  var field1: T
-  var field2: U
-  var field3: V
-}
-
-// The protocol conformance puts a symbol into __DATA_CONST which the builder
-// can use as the base symbol for references to other data.
-public protocol PublicProto {}
-extension Box3: PublicProto {}
+import testMetadataLibrary
 
 let args = CommandLine.arguments
 
@@ -60,8 +39,8 @@ if args.count > 1 && args[1] == "getJSON" {
     GenericField<GenericField<Int8, Int16>,
                  Array<GenericStruct<Double, String, Float>>>.self,
     Array<Array<Array<Array<Array<Array<Array<Double>>>>>>>.self,
-    Box<Int>.self,
-    Box<String>.self,
+    testMetadataLibrary.Box<Int>.self,
+    testMetadataLibrary.Box<String>.self,
     Box3<Int, Int, Int>.self,
   ]
   let typeNames = types.map { _mangledTypeName($0)! }