IRGen: re-enable generate static arrays in read-only data sections.

So far, static arrays had to be put into a writable section, because the isa pointer and the (immortal) ref count field were initialized dynamically at the first use of such an array.

But with a new runtime library, which exports the symbols for the (immortal) ref count field and the isa pointer, it's possible to put the whole array into a read-only section. I.e. make it a constant global.

rdar://94185998

This reverts the revert commit df353ff.
Also, I added a frontend option to disable this optimization: `-disable-readonly-static-objects`

Author: eeckstein

include/swift/AST/ASTContext.h

@@ -880,6 +880,10 @@ class ASTContext final {
   /// for extended existential types.
   AvailabilityContext getParameterizedExistentialRuntimeAvailability();
 
+  /// Get the runtime availability of immortal ref-count symbols, which are
+  /// needed to place array buffers into constant data sections.
+  AvailabilityContext getImmortalRefCountSymbolsAvailability();
+
   /// Get the runtime availability of features introduced in the Swift 5.2
   /// compiler for the target platform.
   AvailabilityContext getSwift52Availability();

include/swift/AST/IRGenOptions.h

@@ -402,6 +402,8 @@ class IRGenOptions {
   /// and protocol conformance caches.
   unsigned NoPreallocatedInstantiationCaches : 1;
 
+  unsigned DisableReadonlyStaticObjects : 1;
+
   /// The number of threads for multi-threaded code generation.
   unsigned NumThreads = 0;
 
@@ -470,7 +472,8 @@ class IRGenOptions {
         EnableGlobalISel(false), VirtualFunctionElimination(false),
         WitnessMethodElimination(false), ConditionalRuntimeRecords(false),
         InternalizeAtLink(false), InternalizeSymbols(false),
-        NoPreallocatedInstantiationCaches(false), CmdArgs(),
+        NoPreallocatedInstantiationCaches(false),
+        DisableReadonlyStaticObjects(false), CmdArgs(),
         SanitizeCoverage(llvm::SanitizerCoverageOptions()),
         TypeInfoFilter(TypeInfoDumpFilter::All) {
 #ifndef NDEBUG

include/swift/IRGen/Linking.h

@@ -364,6 +364,9 @@ class LinkEntity {
     /// A SIL global variable. The pointer is a SILGlobalVariable*.
     SILGlobalVariable,
 
+    /// An outlined read-only global object. The pointer is a SILGlobalVariable*.
+    ReadOnlyGlobalObject,
+
     // These next few are protocol-conformance kinds.
 
     /// A direct protocol witness table. The secondary pointer is a
@@ -993,13 +996,7 @@ class LinkEntity {
     return entity;
   }
 
-  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G) {
-    LinkEntity entity;
-    entity.Pointer = G;
-    entity.SecondaryPointer = nullptr;
-    entity.Data = LINKENTITY_SET_FIELD(Kind, unsigned(Kind::SILGlobalVariable));
-    return entity;
-  }
+  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G, IRGenModule &IGM);
 
   static LinkEntity
   forDifferentiabilityWitness(const SILDifferentiabilityWitness *witness) {
@@ -1420,7 +1417,8 @@ class LinkEntity {
   }
 
   SILGlobalVariable *getSILGlobalVariable() const {
-    assert(getKind() == Kind::SILGlobalVariable);
+    assert(getKind() == Kind::SILGlobalVariable ||
+           getKind() == Kind::ReadOnlyGlobalObject);
     return reinterpret_cast<SILGlobalVariable*>(Pointer);
   }
 

include/swift/Option/FrontendOptions.td

@@ -641,6 +641,10 @@ def disable_preallocated_instantiation_caches : Flag<["-"], "disable-preallocate
   Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>,
   HelpText<"Avoid preallocating metadata instantiation caches in globals">;
 
+def disable_readonly_static_objects : Flag<["-"], "disable-readonly-static-objects">,
+  Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>,
+  HelpText<"Avoid allocating static objects in a read-only data section">;
+
 def trap_function
   : Separate<["-"], "trap-function">, MetaVarName<"<name>">,
     HelpText<"Lower traps to calls to this function instead of trap instructions">;

lib/AST/Availability.cpp

@@ -399,6 +399,13 @@ ASTContext::getParameterizedExistentialRuntimeAvailability() {
   return getSwift57Availability();
 }
 
+AvailabilityContext
+ASTContext::getImmortalRefCountSymbolsAvailability() {
+  // TODO: replace this with a concrete swift version once we have it.
+  // rdar://94185998
+  return getSwiftFutureAvailability();
+}
+
 AvailabilityContext ASTContext::getSwift52Availability() {
   auto target = LangOpts.Target;
 

lib/Frontend/CompilerInvocation.cpp

@@ -2329,6 +2329,10 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
     Opts.NoPreallocatedInstantiationCaches = true;
   }
 
+  if (Args.hasArg(OPT_disable_readonly_static_objects)) {
+    Opts.DisableReadonlyStaticObjects = true;
+  }
+
   // Default to disabling swift async extended frame info on anything but
   // darwin. Other platforms are unlikely to have support for extended frame
   // pointer information.

lib/IRGen/GenConstant.cpp

@@ -278,9 +278,27 @@ llvm::Constant *irgen::emitConstantObject(IRGenModule &IGM, ObjectInst *OI,
     }
   }
   // Construct the object header.
-  llvm::Type *ObjectHeaderTy = sTy->getElementType(0);
-  assert(ObjectHeaderTy->isStructTy());
-  elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  llvm::StructType *ObjectHeaderTy = cast<llvm::StructType>(sTy->getElementType(0));
+
+  if (IGM.canMakeStaticObjectsReadOnly()) {
+    if (!IGM.swiftImmortalRefCount) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.Int8Ty,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftImmortalRefCount");
+      IGM.swiftImmortalRefCount = var;
+    }
+    if (!IGM.swiftStaticArrayMetadata) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.TypeMetadataStructTy,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftStaticArrayMetadata");
+      IGM.swiftStaticArrayMetadata = var;
+    }
+    elts[0] = llvm::ConstantStruct::get(ObjectHeaderTy, {
+      IGM.swiftStaticArrayMetadata,
+      llvm::ConstantExpr::getPtrToInt(IGM.swiftImmortalRefCount, IGM.IntPtrTy)});
+  } else {
+    elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  }
   insertPadding(elts, sTy);
   return llvm::ConstantStruct::get(sTy, elts);
 }

lib/IRGen/GenDecl.cpp

@@ -2482,7 +2482,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
     return Address(addr, alignment);
   }
 
-  LinkEntity entity = LinkEntity::forSILGlobalVariable(var);
+  LinkEntity entity = LinkEntity::forSILGlobalVariable(var, *this);
   ResilienceExpansion expansion = getResilienceExpansionForLayout(var);
 
   llvm::Type *storageType;
@@ -2528,28 +2528,33 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
 
   // Check whether we've created the global variable already.
   // FIXME: We should integrate this into the LinkEntity cache more cleanly.
-  auto gvar = Module.getGlobalVariable(var->getName(), /*allowInternal*/ true);
+  LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
+  auto gvar = Module.getGlobalVariable(link.getName(), /*allowInternal*/ true);
   if (gvar) {
     if (forDefinition)
       updateLinkageForDefinition(*this, gvar, entity);
   } else {
-    LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
     llvm::Type *storageTypeWithContainer = storageType;
     if (var->isInitializedObject()) {
-      // A statically initialized object must be placed into a container struct
-      // because the swift_initStaticObject needs a swift_once_t at offset -1:
-      //     struct Container {
-      //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
-      //       HeapObject object;
-      //     };
-      std::string typeName = storageType->getStructName().str() + 'c';
-      assert(fixedAlignment >= getPointerAlignment());
-      unsigned numTokens = fixedAlignment.getValue() /
-        getPointerAlignment().getValue();
-      storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
-              {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
-      gvar = createVariable(*this, link, storageTypeWithContainer,
-                            fixedAlignment);
+      if (canMakeStaticObjectsReadOnly()) {
+        gvar = createVariable(*this, link, storageType, fixedAlignment);
+        gvar->setConstant(true);
+      } else {
+        // A statically initialized object must be placed into a container struct
+        // because the swift_initStaticObject needs a swift_once_t at offset -1:
+        //     struct Container {
+        //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
+        //       HeapObject object;
+        //     };
+        std::string typeName = storageType->getStructName().str() + 'c';
+        assert(fixedAlignment >= getPointerAlignment());
+        unsigned numTokens = fixedAlignment.getValue() /
+          getPointerAlignment().getValue();
+        storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
+                {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
+        gvar = createVariable(*this, link, storageTypeWithContainer,
+                              fixedAlignment);
+      }
     } else {
       StringRef name;
       Optional<SILLocation> loc;
@@ -2574,7 +2579,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
       gvar->setComdat(nullptr);
   }
   llvm::Constant *addr = gvar;
-  if (var->isInitializedObject()) {
+  if (var->isInitializedObject() && !canMakeStaticObjectsReadOnly()) {
     // Project out the object from the container.
     llvm::Constant *Indices[2] = {
       llvm::ConstantExpr::getIntegerValue(Int32Ty, APInt(32, 0)),

lib/IRGen/IRGenModule.cpp

@@ -1785,6 +1785,19 @@ bool IRGenModule::shouldPrespecializeGenericMetadata() {
          canPrespecializeTarget;
 }
 
+bool IRGenModule::canMakeStaticObjectsReadOnly() {
+  if (getOptions().DisableReadonlyStaticObjects)
+    return false;
+
+  // TODO: Support constant static arrays on other platforms, too.
+  // See also the comment in GlobalObjects.cpp.
+  if (!Triple.isOSDarwin())
+    return false;
+
+  return getAvailabilityContext().isContainedIn(
+          Context.getImmortalRefCountSymbolsAvailability());
+}
+
 void IRGenerator::addGenModule(SourceFile *SF, IRGenModule *IGM) {
   assert(GenModules.count(SF) == 0);
   GenModules[SF] = IGM;

lib/IRGen/IRGenModule.h

@@ -763,6 +763,9 @@ class IRGenModule {
 
   llvm::GlobalVariable *TheTrivialPropertyDescriptor = nullptr;
 
+  llvm::GlobalVariable *swiftImmortalRefCount = nullptr;
+  llvm::GlobalVariable *swiftStaticArrayMetadata = nullptr;
+
   /// Used to create unique names for class layout types with tail allocated
   /// elements.
   unsigned TailElemTypeID = 0;
@@ -886,6 +889,8 @@ class IRGenModule {
 
   bool shouldPrespecializeGenericMetadata();
 
+  bool canMakeStaticObjectsReadOnly();
+  
   Size getAtomicBoolSize() const { return AtomicBoolSize; }
   Alignment getAtomicBoolAlignment() const { return AtomicBoolAlign; }