stdlib, SIL optimizer: use the SIL copy-on-write representation in the Array types.

Use the new builtins for COW representation in Array, ContiguousArray and ArraySlice.
The basic idea is to strictly separate code which mutates an array buffer from code which reads from an array.
The concept is explained in more detail in docs/SIL.rst, section "Copy-on-Write Representation".

The main change is to use beginCOWMutation() instead of isUniquelyReferenced() and insert endCOWMutation() at the end of all mutating functions. Also, reading from the array buffer must be done differently, depending on if the buffer is in a mutable or immutable state.

All the required invariants are enforced by runtime checks - but only in an assert-build of the library: a bit in the buffer object side-table indicates if the buffer is mutable or not.

Along with the library changes, also two optimizations needed to be updated: COWArrayOpt and ObjectOutliner.

Author: eeckstein

lib/SILOptimizer/LoopTransforms/ArrayOpt.h

@@ -118,15 +118,17 @@ class StructUseCollector {
     }
   }
 
-  /// Returns true if there is a single address user of the value.
-  bool hasSingleAddressUse(SILInstruction *SingleAddressUser) {
+  /// Returns true if there are only address users of the value.
+  bool hasOnlyAddressUses(ApplyInst *use1, ApplyInst *use2) {
     if (!AggregateAddressUsers.empty())
       return false;
     if (!ElementAddressUsers.empty())
       return false;
-    if (StructAddressUsers.size() != 1)
-      return false;
-    return StructAddressUsers[0] == SingleAddressUser;
+    for (SILInstruction *user : StructAddressUsers) {
+      if (user != use1 && user != use2)
+        return false;
+    }
+    return true;
   }
 
 protected:

lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp

@@ -436,6 +436,18 @@ bool COWArrayOpt::checkSafeArrayAddressUses(UserList &AddressUsers) {
   return true;
 }
 
+template <typename UserRange>
+ArraySemanticsCall getEndMutationCall(const UserRange &AddressUsers) {
+  for (auto *UseInst : AddressUsers) {
+    if (auto *AI = dyn_cast<ApplyInst>(UseInst)) {
+      ArraySemanticsCall ASC(AI);
+      if (ASC.getKind() == ArrayCallKind::kEndMutation)
+        return ASC;
+    }
+  }
+  return ArraySemanticsCall();
+}
+
 /// Returns true if this instruction is a safe array use if all of its users are
 /// also safe array users.
 static SILValue isTransitiveSafeUser(SILInstruction *I) {
@@ -811,8 +823,14 @@ void COWArrayOpt::hoistAddressProjections(Operand &ArrayOp) {
   }
 }
 
-/// Check if this call to "make_mutable" is hoistable, and move it, or delete it
-/// if it's already hoisted.
+/// Check if this call to "make_mutable" is hoistable, and copy it, along with
+/// the corresponding end_mutation call, to the loop pre-header.
+///
+/// The origial make_mutable/end_mutation calls remain in the loop, because
+/// removing them would violate the COW representation rules.
+/// Having those calls in the pre-header will then enable COWOpts (after
+/// inlining) to constant fold the uniqueness check of the begin_cow_mutation
+/// in the loop.
 bool COWArrayOpt::hoistMakeMutable(ArraySemanticsCall MakeMutable,
                                    bool dominatesExits) {
   LLVM_DEBUG(llvm::dbgs() << "    Checking mutable array: " <<CurrentArrayAddr);
@@ -872,6 +890,18 @@ bool COWArrayOpt::hoistMakeMutable(ArraySemanticsCall MakeMutable,
       return false;
   }
 
+  auto ArrayUsers = llvm::map_range(MakeMutable.getSelf()->getUses(),
+                                    ValueBase::UseToUser());
+
+  // There should be a call to end_mutation. Find it so that we can copy it to
+  // the pre-header.
+  ArraySemanticsCall EndMutation = getEndMutationCall(ArrayUsers);
+  if (!EndMutation) {
+    EndMutation = getEndMutationCall(StructUses.StructAddressUsers);
+    if (!EndMutation)
+      return false;
+  }
+
   // Hoist the make_mutable.
   LLVM_DEBUG(llvm::dbgs() << "    Hoisting make_mutable: " << *MakeMutable);
 
@@ -880,12 +910,18 @@ bool COWArrayOpt::hoistMakeMutable(ArraySemanticsCall MakeMutable,
   assert(MakeMutable.canHoist(Preheader->getTerminator(), DomTree) &&
          "Should be able to hoist make_mutable");
 
-  MakeMutable.hoist(Preheader->getTerminator(), DomTree);
+  // Copy the make_mutable and end_mutation calls to the pre-header.
+  TermInst *insertionPoint = Preheader->getTerminator();
+  ApplyInst *hoistedMM = MakeMutable.copyTo(insertionPoint, DomTree);
+  ApplyInst *EMInst = EndMutation;
+  ApplyInst *hoistedEM = cast<ApplyInst>(EMInst->clone(insertionPoint));
+  hoistedEM->setArgument(0, hoistedMM->getArgument(0));
+  placeFuncRef(hoistedEM, DomTree);
 
   // Register array loads. This is needed for hoisting make_mutable calls of
   // inner arrays in the two-dimensional case.
   if (arrayContainerIsUnique &&
-      StructUses.hasSingleAddressUse((ApplyInst *)MakeMutable)) {
+      StructUses.hasOnlyAddressUses((ApplyInst *)MakeMutable, EMInst)) {
     for (auto use : MakeMutable.getSelf()->getUses()) {
       if (auto *LI = dyn_cast<LoadInst>(use->getUser()))
         HoistableLoads.insert(LI);
@@ -917,39 +953,33 @@ bool COWArrayOpt::run() {
   // is only mapped to a call once the analysis has determined that no
   // make_mutable calls are required within the loop body for that array.
   llvm::SmallDenseMap<SILValue, ApplyInst*> ArrayMakeMutableMap;
-
+  
+  llvm::SmallVector<ArraySemanticsCall, 8> makeMutableCalls;
+  
   for (auto *BB : Loop->getBlocks()) {
     if (ColdBlocks.isCold(BB))
       continue;
-    bool dominatesExits = dominatesExitingBlocks(BB);
-    for (auto II = BB->begin(), IE = BB->end(); II != IE;) {
-      // Inst may be moved by hoistMakeMutable.
-      SILInstruction *Inst = &*II;
-      ++II;
-      ArraySemanticsCall MakeMutableCall(Inst, "array.make_mutable");
-      if (!MakeMutableCall)
-        continue;
+      
+    // Instructions are getting moved around. To not mess with iterator
+    // invalidation, first collect all calls, and then do the transformation.
+    for (SILInstruction &I : *BB) {
+      ArraySemanticsCall MakeMutableCall(&I, "array.make_mutable");
+      if (MakeMutableCall)
+        makeMutableCalls.push_back(MakeMutableCall);
+    }
 
+    bool dominatesExits = dominatesExitingBlocks(BB);
+    for (ArraySemanticsCall MakeMutableCall : makeMutableCalls) {
       CurrentArrayAddr = MakeMutableCall.getSelf();
       auto HoistedCallEntry = ArrayMakeMutableMap.find(CurrentArrayAddr);
       if (HoistedCallEntry == ArrayMakeMutableMap.end()) {
-        if (!hoistMakeMutable(MakeMutableCall, dominatesExits)) {
+        if (hoistMakeMutable(MakeMutableCall, dominatesExits)) {
+          ArrayMakeMutableMap[CurrentArrayAddr] = MakeMutableCall;
+          HasChanged = true;
+        } else {
           ArrayMakeMutableMap[CurrentArrayAddr] = nullptr;
-          continue;
         }
-
-        ArrayMakeMutableMap[CurrentArrayAddr] = MakeMutableCall;
-        HasChanged = true;
-        continue;
       }
-
-      if (!HoistedCallEntry->second)
-        continue;
-
-      LLVM_DEBUG(llvm::dbgs() << "    Removing make_mutable call: "
-                              << *MakeMutableCall);
-      MakeMutableCall.removeCall();
-      HasChanged = true;
     }
   }
   return HasChanged;