[Caching] Allow prefix mapping on legacy layout file

To support distributed caching for targets that need to use legacy
layout file, the path of legacy layout needs to be remapped.

Current handling of the legacy layout file is to ingest all layout files
to the CAS FileSystem as part of the compiler resource files. But it
cannot convey remapped legacy layout file path to the swift-frontend
when distributed caching is enabled. In order to properly support path
remapping, the legacy layout file is ingested on demand (thus it doesn't
need to be ingested for module compilation), and the remapped path is
communicated to swift-front via frontend flag.

rdar://162793678

Author: cachemeifyoucan

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -145,7 +145,11 @@ class SwiftDependencyTracker {
                          const CompilerInvocation &CI);
 
   void startTracking(bool includeCommonDeps = true);
-  void trackFile(const Twine &path);
+
+  /// Track a file with path.
+  /// \returns true if the file is tracked, false if the file doesn't exist.
+  bool trackFile(const Twine &path);
+
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -452,20 +452,6 @@ SwiftDependencyTracker::SwiftDependencyTracker(
                           "SDKSettings.json");
   addCommonFile(SDKSettingPath);
 
-  // Add Legacy layout file.
-  const std::vector<std::string> AllSupportedArches = {
-      "arm64", "arm64e", "x86_64", "i386",
-      "armv7", "armv7s", "armv7k", "arm64_32"};
-
-  for (auto RuntimeLibPath : SearchPathOpts.RuntimeLibraryPaths) {
-    std::error_code EC;
-    for (auto &Arch : AllSupportedArches) {
-      SmallString<256> LayoutFile(RuntimeLibPath);
-      llvm::sys::path::append(LayoutFile, "layouts-" + Arch + ".yaml");
-      addCommonFile(LayoutFile);
-    }
-  }
-
   // Add VFSOverlay file.
   for (auto &Overlay: SearchPathOpts.VFSOverlayFiles)
     addCommonFile(Overlay);
@@ -488,20 +474,21 @@ void SwiftDependencyTracker::startTracking(bool includeCommonDeps) {
   }
 }
 
-void SwiftDependencyTracker::trackFile(const Twine &path) {
+bool SwiftDependencyTracker::trackFile(const Twine &path) {
   auto file = FS->openFileForRead(path);
   if (!file)
-    return;
+    return false;
   auto status = (*file)->status();
   if (!status)
-    return;
+    return false;
   auto CASFile = dyn_cast<llvm::cas::CASBackedFile>(*file);
   if (!CASFile)
-    return;
+    return false;
   auto fileRef = CASFile->getObjectRefForContent();
   std::string realPath =
       Mapper ? Mapper->mapToString(path.str()) : path.str();
   TrackedFiles.try_emplace(realPath, fileRef, (size_t)status->getSize());
+  return true;
 }
 
 llvm::Expected<llvm::cas::ObjectProxy>

lib/DependencyScan/ScanDependencies.cpp

@@ -471,6 +471,27 @@ class ExplicitModuleDependencyResolver {
                      [this](const auto &entry) {
                        tracker->trackFile(entry.second.LibraryPath);
                      });
+      StringRef readLegacyTypeInfoPath =
+          instance.getInvocation().getIRGenOptions().ReadLegacyTypeInfoPath;
+      if (!readLegacyTypeInfoPath.empty()) {
+        // If legacy layout is specifed, just need to track that file.
+        if (tracker->trackFile(readLegacyTypeInfoPath))
+          commandline.push_back("-read-legacy-type-info-path=" +
+                                scanner.remapPath(readLegacyTypeInfoPath));
+      } else {
+        // Otherwise, search RuntimeLibrary Path for implicit legacy layout.
+        // Search logic need to match IRGen/GenType.cpp.
+        const auto &triple = instance.getInvocation().getLangOptions().Target;
+        auto archName = swift::getMajorArchitectureName(triple);
+        SmallString<256> legacyLayoutPath(instance.getInvocation()
+                                              .getSearchPathOptions()
+                                              .RuntimeLibraryPaths[0]);
+        llvm::sys::path::append(legacyLayoutPath,
+                                "layouts-" + archName + ".yaml");
+        if (tracker->trackFile(legacyLayoutPath))
+          commandline.push_back("-read-legacy-type-info-path=" +
+                                scanner.remapPath(legacyLayoutPath));
+      }
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return diagnoseCASFSCreationError(root.takeError());

test/CAS/deps_cas_fs.swift

@@ -5,7 +5,7 @@
 // RUN: mkdir -p %t/cas
 
 // RUN: mkdir -p %t/resource/macosx
-// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-x86_64.yaml
+// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-%target-arch.yaml
 
 // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -cache-compile-job -cas-path %t/cas -module-name Test -resource-dir %t/resource -scanner-output-dir %t
 // Check the contents of the JSON output
@@ -19,13 +19,13 @@
 // RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Test casFSRootID > %t/Test_fs.casid
 // RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/Test_fs.casid | %FileCheck %s -check-prefix FS_ROOT_TEST
 
-// FS_ROOT_E-DAG: layouts-x86_64.yaml
-// FS_ROOT_E-DAG: E.swiftinterface
+// FS_ROOT_E-NOT: layouts-{{.*}}.yaml
+// FS_ROOT_E: E.swiftinterface
 
-// FS_ROOT_F-DAG: layouts-x86_64.yaml
-// FS_ROOT_F-DAG: F.swiftinterface
+// FS_ROOT_F-NOT: layouts-{{.*}}.yaml
+// FS_ROOT_F: F.swiftinterface
 
-// FS_ROOT_TEST-DAG: layouts-x86_64.yaml
+// FS_ROOT_TEST-DAG: layouts-{{.*}}.yaml
 // FS_ROOT_TEST-DAG: deps_cas_fs.swift
 
 import E

test/CAS/legacy_layout_remap.swift

@@ -0,0 +1,73 @@
+// REQUIRES: objc_interop
+// UNSUPPORTED: swift_only_stable_abi
+
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+/// Build legacy module.
+// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -emit-module -enable-library-evolution \
+// RUN:   -emit-module-path=%t/resilient_struct.swiftmodule -module-name=resilient_struct -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %S/../Inputs/resilient_struct.swift
+
+/// Scan with legacy layout.
+// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution -read-legacy-type-info-path=%t/layout.yaml \
+// RUN:   -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/deps.json -swift-version 4 -cache-compile-job -cas-path %t/cas \
+// RUN:   -scanner-prefix-map-paths %t /^tmp
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd
+// RUN: %swift_frontend_plain @%t/shim.cmd
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+
+// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \
+// RUN:   -cache-compile-job -cas-path %t/cas -O \
+// RUN:   -swift-version 4 -disable-implicit-swift-modules \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name Test -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   /^tmp/main.swift @%t/MyApp.cmd -c -o %t/main.o
+
+/// Now do implicit search.
+// RUN: mkdir -p %t/resource/macosx
+// RUN: cp %t/layout.yaml %t/resource/macosx/layouts-%target-arch.yaml
+
+// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution \
+// RUN:   -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/deps2.json -swift-version 4 -cache-compile-job -cas-path %t/cas \
+// RUN:   -scanner-prefix-map-paths %t /^tmp -resource-dir %t/resource -I %platform-dylib-dir
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json clang:SwiftShims > %t/shim2.cmd
+// RUN: %swift_frontend_plain @%t/shim2.cmd
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps2.json > %t/map2.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map2.json > %t/map2.casid
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json Test > %t/MyApp2.cmd
+
+// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \
+// RUN:   -cache-compile-job -cas-path %t/cas -O \
+// RUN:   -swift-version 4 -disable-implicit-swift-modules \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name Test -explicit-swift-module-map-file @%t/map2.casid \
+// RUN:   /^tmp/main.swift @%t/MyApp2.cmd -c -o %t/main.o
+
+//--- main.swift
+import resilient_struct
+
+public class ClassWithResilientRef {
+  var first: ResilientRef? = nil
+  var second: Int = 0
+}
+
+//--- layout.yaml
+Name:            resilient_struct
+Decls:
+  - Name:            16resilient_struct12ResilientRefV
+    Size:            8
+    Alignment:       8
+    ExtraInhabitants: 4096