Merge pull request #84110 from DougGregor/lazy-irgen-global-variables

[IRGen] Lazily emit SIL global variables

Author: DougGregor

lib/IRGen/GenDecl.cpp

@@ -1174,6 +1174,21 @@ static bool isLazilyEmittedFunction(SILFunction &f, SILModule &m) {
   return true;
 }
 
+// Eagerly emit global variables that are externally visible.
+static bool isLazilyEmittedGlobalVariable(SILGlobalVariable &v, SILModule &m) {
+  if (v.isPossiblyUsedExternally()) {
+    // Under the embedded linkage model, if it has a non-unique definition,
+    // treat it lazily.
+    if (v.hasNonUniqueDefinition() && !v.markedAsUsed()) {
+      return true;
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
 void IRGenerator::emitGlobalTopLevel(
     const std::vector<std::string> &linkerDirectives) {
   if (PrimaryIGM->getSILModule().getOptions().StopOptimizationAfterSerialization) {
@@ -1207,6 +1222,9 @@ void IRGenerator::emitGlobalTopLevel(
     createLinkerDirectiveVariable(*PrimaryIGM, directive);
   }
   for (SILGlobalVariable &v : PrimaryIGM->getSILModule().getSILGlobals()) {
+    if (isLazilyEmittedGlobalVariable(v, PrimaryIGM->getSILModule()))
+      continue;
+
     Decl *decl = v.getDecl();
     CurrentIGMPtr IGM = getGenModule(decl ? decl->getDeclContext() : nullptr);
     IGM->emitSILGlobalVariable(&v);
@@ -1368,6 +1386,7 @@ void IRGenerator::emitLazyDefinitions() {
          !LazyExtensionDescriptors.empty() ||
          !LazyFieldDescriptors.empty() ||
          !LazyFunctionDefinitions.empty() || !LazyWitnessTables.empty() ||
+         !LazyGlobalVariables.empty() ||
          !LazyCanonicalSpecializedMetadataAccessors.empty() ||
          !LazyMetadataAccessors.empty() ||
          !LazyClassMetadata.empty() ||
@@ -1456,6 +1475,13 @@ void IRGenerator::emitLazyDefinitions() {
       IGM->emitSILFunction(f);
     }
 
+    // Emit any lazy global variables we require.
+    while (!LazyGlobalVariables.empty()) {
+      SILGlobalVariable *v = LazyGlobalVariables.pop_back_val();
+      CurrentIGMPtr IGM = getGenModule(v);
+      IGM->emitSILGlobalVariable(v);
+    }
+
     while (!LazyCanonicalSpecializedMetadataAccessors.empty()) {
       CanType theType =
           LazyCanonicalSpecializedMetadataAccessors.pop_back_val();
@@ -1529,6 +1555,26 @@ void IRGenerator::addLazyFunction(SILFunction *f) {
   DefaultIGMForFunction.insert(std::make_pair(f, CurrentIGM));
 }
 
+void IRGenerator::addLazyGlobalVariable(SILGlobalVariable *v) {
+  // Add it to the queue if it hasn't already been put there.
+  if (!LazilyEmittedGlobalVariables.insert(v).second)
+    return;
+
+  assert(!FinishedEmittingLazyDefinitions);
+  LazyGlobalVariables.push_back(v);
+
+  if (auto decl = v->getDecl()) {
+    if (decl->getDeclContext()->getParentSourceFile())
+      return;
+  }
+
+  if (CurrentIGM == nullptr)
+    return;
+
+  // Don't update the map if we already have an entry.
+  DefaultIGMForGlobalVariable.insert(std::make_pair(v, CurrentIGM));
+}
+
 bool IRGenerator::hasLazyMetadata(TypeDecl *type) {
   assert(isa<NominalTypeDecl>(type) ||
          isa<OpaqueTypeDecl>(type));
@@ -2714,6 +2760,9 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
     auto alignment =
       Alignment(getClangASTContext().getDeclAlign(clangDecl).getQuantity());
     return Address(addr, ti.getStorageType(), alignment);
+  } else if (!forDefinition &&
+             isLazilyEmittedGlobalVariable(*var, getSILModule())) {
+    IRGen.addLazyGlobalVariable(var);
   }
 
   ResilienceExpansion expansion = getResilienceExpansionForLayout(var);
@@ -2726,7 +2775,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
 
   if (var->isInitializedObject()) {
     assert(ti.isFixedSize(expansion));
-    StructLayout *Layout = StaticObjectLayouts[var].get();
+    StructLayout *Layout = StaticObjectLayouts[var].layout.get();
     if (!Layout) {
       // Create the layout (includes the llvm type) for the statically
       // initialized object and store it for later.
@@ -2737,7 +2786,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
       }
       Layout = getClassLayoutWithTailElems(*this,
                                            var->getLoweredType(), TailTypes);
-      StaticObjectLayouts[var] = std::unique_ptr<StructLayout>(Layout);
+      StaticObjectLayouts[var] = {std::unique_ptr<StructLayout>(Layout), nullptr};
     }
     storageType = Layout->getType();
     fixedSize = Layout->getSize();
@@ -2853,7 +2902,7 @@ llvm::Constant *IRGenModule::getGlobalInitValue(SILGlobalVariable *var,
                                                 llvm::Type *storageType,
                                                 Alignment alignment) {
   if (var->isInitializedObject()) {
-    StructLayout *layout = StaticObjectLayouts[var].get();
+    StructLayout *layout = StaticObjectLayouts[var].layout.get();
     ObjectInst *oi = cast<ObjectInst>(var->getStaticInitializerValue());
     llvm::Constant *initVal = emitConstantObject(*this, oi, layout);
     if (!canMakeStaticObjectReadOnly(var->getLoweredType())) {
@@ -2863,13 +2912,18 @@ llvm::Constant *IRGenModule::getGlobalInitValue(SILGlobalVariable *var,
       //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
       //       HeapObject object;
       //     };
-      std::string typeName = storageType->getStructName().str() + 'c';
-      assert(alignment >= getPointerAlignment());
-      unsigned numTokens = alignment.getValue() /
-                           getPointerAlignment().getValue();
-      auto *containerTy = llvm::StructType::create(getLLVMContext(),
-              {llvm::ArrayType::get(OnceTy, numTokens), initVal->getType()},
-              typeName);
+      llvm::StructType *containerTy = StaticObjectLayouts[var].containerTy;
+      if (!containerTy) {
+        std::string typeName = storageType->getStructName().str() + 'c';
+        assert(alignment >= getPointerAlignment());
+        unsigned numTokens = alignment.getValue() /
+                             getPointerAlignment().getValue();
+        containerTy = llvm::StructType::create(getLLVMContext(),
+                {llvm::ArrayType::get(OnceTy, numTokens), initVal->getType()},
+                typeName);
+        StaticObjectLayouts[var].containerTy = containerTy;
+      }
+
       auto *zero = llvm::ConstantAggregateZero::get(containerTy->getElementType(0));
       initVal = llvm::ConstantStruct::get(containerTy, {zero , initVal});
     }

lib/IRGen/IRGenModule.cpp

@@ -2300,6 +2300,22 @@ IRGenModule *IRGenerator::getGenModule(SILFunction *f) {
   return getPrimaryIGM();
 }
 
+IRGenModule *IRGenerator::getGenModule(SILGlobalVariable *v) {
+  if (GenModules.size() == 1) {
+    return getPrimaryIGM();
+  }
+
+  auto found = DefaultIGMForGlobalVariable.find(v);
+  if (found != DefaultIGMForGlobalVariable.end())
+    return found->second;
+
+  if (auto decl = v->getDecl()) {
+    return getGenModule(decl->getDeclContext());
+  }
+
+  return getPrimaryIGM();
+}
+
 uint32_t swift::irgen::getSwiftABIVersion() {
   return IRGenModule::swiftVersion;
 }

lib/IRGen/IRGenModule.h

@@ -228,6 +228,10 @@ class IRGenerator {
   // It is used if a function has no source-file association.
   llvm::DenseMap<SILFunction *, IRGenModule *> DefaultIGMForFunction;
 
+  // Stores the IGM from which a global variable is referenced the first time.
+  // It is used if a global variable has no source-file association.
+  llvm::DenseMap<SILGlobalVariable *, IRGenModule *> DefaultIGMForGlobalVariable;
+
   // The IGMs where specializations of functions are emitted. The key is the
   // non-specialized function.
   // Storing all specializations of a function in the same IGM increases the
@@ -327,6 +331,13 @@ class IRGenerator {
   /// The queue of SIL functions to emit.
   llvm::SmallVector<SILFunction *, 4> LazyFunctionDefinitions;
 
+  /// SIL global variables that have already been lazily emitted, or are
+  /// queued up.
+  llvm::SmallPtrSet<SILGlobalVariable *, 4> LazilyEmittedGlobalVariables;
+
+  /// The queue of SIL global variables to emit.
+  llvm::SmallVector<SILGlobalVariable *, 4> LazyGlobalVariables;
+
   /// Witness tables that have already been lazily emitted, or are queued up.
   llvm::SmallPtrSet<SILWitnessTable *, 4> LazilyEmittedWitnessTables;
 
@@ -388,6 +399,12 @@ class IRGenerator {
   /// first time.
   IRGenModule *getGenModule(SILFunction *f);
 
+  /// Get an IRGenModule for a global variable.
+  /// Returns the IRGenModule of the containing source file, or if this cannot
+  /// be determined, returns the IGM from which the global variable is
+  /// referenced the first time.
+  IRGenModule *getGenModule(SILGlobalVariable *v);
+
   /// Returns the primary IRGenModule. This is the first added IRGenModule.
   /// It is used for everything which cannot be correlated to a specific source
   /// file. And of course, in single-threaded compilation there is only the
@@ -454,6 +471,8 @@ class IRGenerator {
 
   void addLazyFunction(SILFunction *f);
 
+  void addLazyGlobalVariable(SILGlobalVariable *v);
+
   void addDynamicReplacement(SILFunction *f) { DynamicReplacements.insert(f); }
 
   void forceLocalEmitOfLazyFunction(SILFunction *f) {
@@ -1407,8 +1426,14 @@ class IRGenModule {
   friend struct ::llvm::DenseMapInfo<swift::irgen::IRGenModule::FixedLayoutKey>;
   llvm::DenseMap<FixedLayoutKey, llvm::Constant *> PrivateFixedLayouts;
 
+  /// Captures a static object layout.
+  struct StaticObjectLayout {
+    std::unique_ptr<StructLayout> layout;
+    llvm::StructType *containerTy = nullptr;
+  };
+
   /// A cache for layouts of statically initialized objects.
-  llvm::DenseMap<SILGlobalVariable *, std::unique_ptr<StructLayout>>
+  llvm::DenseMap<SILGlobalVariable *, StaticObjectLayout>
     StaticObjectLayouts;
 
   /// A mapping from order numbers to the LLVM functions which we

lib/SIL/IR/SILGlobalVariable.cpp

@@ -84,6 +84,9 @@ bool SILGlobalVariable::isPossiblyUsedExternally() const {
   if (shouldBePreservedForDebugger())
     return true;
 
+  if (markedAsUsed())
+    return true;
+
   SILLinkage linkage = getLinkage();
   return swift::isPossiblyUsedExternally(linkage, getModule().isWholeModule());
 }

test/IRGen/constant_struct_with_padding.sil

@@ -11,7 +11,7 @@ struct T {
 }
 
 // CHECK: @global = hidden global %T4main1TV <{ i1 false, [3 x i8] undef, i32 0 }>, align 4
-sil_global hidden @global : $T = {
+sil_global hidden [used] @global : $T = {
   %2 = integer_literal $Builtin.Int1, 0
   %3 = integer_literal $Builtin.Int32, 0
   %initval = struct $T (%2 : $Builtin.Int1, %3 : $Builtin.Int32)

test/IRGen/global_resilience.sil

@@ -39,14 +39,6 @@ public struct LargeResilientStruct {
 // CHECK: @largeGlobal = {{(dllexport )?}}{{(protected )?}}global [[BUFFER]] zeroinitializer
 sil_global [let] @largeGlobal : $LargeResilientStruct
 
-//
-// Size is known in this resilience domain, and global is hidden,
-// so allocate it directly.
-//
-
-// CHECK: @fixedGlobal = hidden global %T17global_resilience20LargeResilientStructV zeroinitializer
-sil_global hidden @fixedGlobal : $LargeResilientStruct
-
 //
 // Unknown size -- must call value witness functions for buffer
 // management.
@@ -55,6 +47,14 @@ sil_global hidden @fixedGlobal : $LargeResilientStruct
 // CHECK: @otherGlobal = {{(dllexport )?}}{{(protected )?}}global [[BUFFER]] zeroinitializer
 sil_global [let] @otherGlobal : $Size
 
+//
+// Size is known in this resilience domain, and global is hidden,
+// so allocate it directly.
+//
+
+// CHECK: @fixedGlobal = hidden global %T17global_resilience20LargeResilientStructV zeroinitializer
+sil_global hidden @fixedGlobal : $LargeResilientStruct
+
 // CHECK-LABEL: define{{( dllexport)?}}{{( protected)?}} swiftcc i32 @testSmallGlobal()
 sil @testSmallGlobal : $@convention(thin) () -> Int32 {
 bb0:

test/IRGen/lazy_globals.swift

@@ -1,11 +1,11 @@
 // RUN: %target-swift-frontend -parse-as-library -emit-ir -primary-file %s | %FileCheck %s
 
-// REQUIRES: CPU=x86_64
+// REQUIRES: CPU=x86_64 || CPU=arm64
 
-// CHECK: @"[[T:.*]]Wz" = internal global i64 0, align 8
 // CHECK: @"$s12lazy_globals1xSivp" = hidden global %TSi zeroinitializer, align 8
 // CHECK: @"$s12lazy_globals1ySivp" = hidden global %TSi zeroinitializer, align 8
 // CHECK: @"$s12lazy_globals1zSivp" = hidden global %TSi zeroinitializer, align 8
+// CHECK: @"[[T:.*]]Wz" = internal global i64 0, align 8
 
 // CHECK: define internal void @"[[T]]WZ"(ptr %0) {{.*}} {
 // CHECK: entry:
@@ -39,4 +39,3 @@ var (x, y, z) = (1, 2, 3)
 // CHECK:   ret i64 [[V]]
 // CHECK: }
 func getX() -> Int { return x }
-

test/embedded/linkage/leaf_application.swift

@@ -21,9 +21,10 @@
 
 //--- Library.swift
 
-// TODO: Once global variables can be emitted lazily, these should be -NOT
-// again, then show up in the application binary if we use them.
-// LIBRARY-IR: @"$es23_swiftEmptyArrayStorageSi_S3itvp" = weak_odr {{(protected |dllexport )?}}global
+// Never referenced.
+// LIBRARY-IR-NOT: @"$es23_swiftEmptyArrayStorageSi_S3itvp" = weak_odr {{(protected |dllexport )?}}global
+
+// Note: referenced by swift_allocEmptyBox.
 // LIBRARY-IR: @"$es16_emptyBoxStorageSi_Sitvp" = weak_odr {{(protected |dllexport )?}}global
 
 // LIBRARY-IR-NOT: define {{.*}}@"$e7Library5helloSaySiGyF"()

test/embedded/modules-globals.swift

@@ -33,4 +33,4 @@ public func main() {
 // CHECK: @"$e4Main022global_in_client_used_c1_D0Sivp" = {{.*}}global %TSi zeroinitializer
 // CHECK: @"$e4Main024global_in_client_unused_c1_D0Sivp" = {{.*}}global %TSi zeroinitializer
 // CHECK: @"$e8MyModule022global_in_module_used_d1_E0Sivp" = {{.*}}global %TSi zeroinitializer
-// CHECK: @"$e8MyModule024global_in_module_unused_d1_E0Sivp" = {{.*}}global %TSi zeroinitializer
+// CHECK-NOT: @"$e8MyModule024global_in_module_unused_d1_E0Sivp" = {{.*}}global %TSi zeroinitializer