[Mem2Reg] Owned lexical lifetimes get moves.

Author: nate-chandler

lib/SILOptimizer/Transforms/SILMem2Reg.cpp

@@ -74,21 +74,20 @@ class LiveValues {
 public:
   struct Owned {
     SILValue stored = SILValue();
-    SILValue borrow = SILValue();
-    SILValue copy = SILValue();
+    SILValue move = SILValue();
 
     /// Create an instance of the minimum values required to replace a usage of
     /// an AllocStackInst.  It consists of only one value.
     ///
-    /// Whether the one value occupies the stored or the copy field depends on
+    /// Whether the one value occupies the stored or the move field depends on
     /// whether the alloc_stack is lexical.  If it is lexical, then usages of
-    /// the asi will be replaced with usages of the copy field; otherwise, those
-    /// usages will be replaced with usages of the stored field.  The
+    /// the asi will be replaced with usages of the move field; otherwise,
+    /// those usages will be replaced with usages of the stored field.  The
     /// implementation constructs an instance to match those requirements.
     static Owned toReplace(AllocStackInst *asi, SILValue replacement) {
       if (lexicalLifetimeEnsured(asi))
-        return {SILValue(), SILValue(), replacement};
-      return {replacement, SILValue(), SILValue()};
+        return {SILValue(), replacement};
+      return {replacement, SILValue()};
     }
 
     /// The value with which usages of the provided AllocStackInst should be
@@ -97,13 +96,9 @@ class LiveValues {
       if (!lexicalLifetimeEnsured(asi)) {
         return stored;
       }
-      // We should have created a borrow of the @owned stored value and a copy
-      // of the borrowed value.
-      assert(copy && borrow);
-      if (isa<LoadBorrowInst>(toReplace)) {
-        return borrow ? borrow : stored;
-      }
-      return copy ? copy : borrow ? borrow : stored;
+      // We should have created a move of the @owned stored value.
+      assert(move);
+      return move;
     }
 
     bool canEndLexicalLifetime() {
@@ -112,7 +107,7 @@ class LiveValues {
       // to end a lexical lifetime. In that case, the lifetime end will be
       // added later, when we have enough information, namely the live in
       // values, to end it.
-      return borrow;
+      return move;
     }
   };
   struct Guaranteed {
@@ -195,8 +190,8 @@ class LiveValues {
     return LiveValues::forGuaranteed({stored, borrow});
   }
 
-  static LiveValues forOwned(SILValue stored, SILValue borrow, SILValue copy) {
-    return LiveValues::forOwned({stored, borrow, copy});
+  static LiveValues forOwned(SILValue stored, SILValue move) {
+    return LiveValues::forOwned({stored, move});
   }
 
   static LiveValues toReplace(AllocStackInst *asi, SILValue replacement) {
@@ -427,8 +422,8 @@ replaceLoad(SILInstruction *inst, SILValue newValue, AllocStackInst *asi,
   op = inst->getOperand(0);
 
   if (auto *lbi = dyn_cast<LoadBorrowInst>(inst)) {
-    if (lexicalLifetimeEnsured(asi)) {
-      assert(newValue->getOwnershipKind() == OwnershipKind::Guaranteed);
+    if (lexicalLifetimeEnsured(asi) &&
+        newValue->getOwnershipKind() == OwnershipKind::Guaranteed) {
       SmallVector<SILInstruction *, 4> endBorrows;
       for (auto *ebi : lbi->getUsersOfType<EndBorrowInst>()) {
         endBorrows.push_back(ebi);
@@ -517,23 +512,22 @@ static bool canEndLexicalLifetime(LiveValues values) {
 ///
 /// The beginning of the scope looks like
 ///
-///     %lifetime = begin_borrow [lexical] %original
-///     %copy = copy_value %lifetime
+///     %lifetime = move_value [lexical] %original
+///
+/// Because the value was consumed by the original store instruction, it can
+/// be rewritten to be consumed by a lexical move_value.
 static StorageStateTracking<LiveValues>
 beginOwnedLexicalLifetimeAfterStore(AllocStackInst *asi, StoreInst *inst) {
   assert(lexicalLifetimeEnsured(asi));
   SILValue stored = inst->getOperand(CopyLikeInstruction::Src);
   SILLocation loc = RegularLocation::getAutoGeneratedLocation(inst->getLoc());
 
-  BeginBorrowInst *bbi = nullptr;
-  CopyValueInst *cvi = nullptr;
+  MoveValueInst *mvi = nullptr;
   SILBuilderWithScope::insertAfter(inst, [&](SILBuilder &builder) {
-    bbi = builder.createBeginBorrow(loc, stored, /*isLexical*/ true);
-    cvi = builder.createCopyValue(loc, bbi);
+    mvi = builder.createMoveValue(loc, stored, /*isLexical*/ true);
   });
-  StorageStateTracking<LiveValues> vals = {
-      LiveValues::forOwned(stored, bbi, cvi),
-      /*isStorageValid=*/true};
+  StorageStateTracking<LiveValues> vals = {LiveValues::forOwned(stored, mvi),
+                                           /*isStorageValid=*/true};
   return vals;
 }
 
@@ -560,26 +554,22 @@ beginGuaranteedLexicalLifetimeAfterStore(AllocStackInst *asi,
 ///
 /// The end of the scope looks like
 ///
-///     end_borrow %lifetime
-///     destroy_value %original
+///     destroy_value %lifetime
+///
+/// This instruction corresponds to the following instructions that begin a
+/// lexical borrow scope:
 ///
-/// These two instructions correspond to the following instructions that begin
-/// a lexical borrow scope:
+///     %lifetime = move_value [lexical] %original
 ///
-///     %lifetime = begin_borrow %original
-///     %copy = copy_value %lifetime
+/// However, no intervention is required to explicitly end the lifetime because
+/// it will already have been ended naturally by destroy_addrs (or equivalent)
+/// of the alloc_stack.
 static void endOwnedLexicalLifetimeBeforeInst(AllocStackInst *asi,
                                               SILInstruction *beforeInstruction,
                                               SILBuilderContext &ctx,
                                               LiveValues::Owned values) {
   assert(lexicalLifetimeEnsured(asi));
   assert(beforeInstruction);
-
-  auto builder = SILBuilderWithScope(beforeInstruction, ctx);
-  SILLocation loc =
-      RegularLocation::getAutoGeneratedLocation(beforeInstruction->getLoc());
-  builder.createEndBorrow(loc, values.borrow);
-  builder.createDestroyValue(loc, values.stored);
 }
 
 /// End the lexical borrow scope for an @guaranteed stored value described by
@@ -647,12 +637,26 @@ class StackAllocationPromoter {
   ///
   /// The live-out values for every block can be derived from these.
   ///
-  /// For non-lexical alloc_stacks, that is just a StoreInst or a
-  /// StoreBorrowInst.
-  /// For lexical alloc_stacks, that is the StoreInst, a BeginBorrowInst of the
-  /// value stored into the StoreInst and a CopyValueInst of the result of the
-  /// BeginBorrowInst or a StoreBorrowInst, and a BeginBorrowInst of the stored
-  /// value if it did not have a guaranteed lexical scope.
+  /// This is either a StoreInst or a StoreBorrowInst.
+  ///
+  /// If the alloc_stack is non-lexical, the only live-out value is the source
+  /// operand of the instruction.
+  ///
+  /// If the alloc_stack is lexical but the stored value is already lexical, no
+  /// additional lexical lifetime is necessary and as an optimization can be
+  /// omitted.  In that case, the only live-out value is the source operand of
+  /// the instruction.  This optimization has been implemented for guaranteed
+  /// alloc_stacks.
+  ///
+  /// If the alloc_stack is lexical and the stored value is not already lexical,
+  /// a lexical lifetime must be introduced that matches the duration in which
+  /// the value remains in the alloc_stack:
+  /// - For owned alloc_stacks, a move_value [lexical] of the stored value is
+  /// created.  That move_value is the instruction after the store, and it is
+  /// the other running value.
+  /// - For guaranteed alloc_stacks, a begin_borrow [lexical] of the
+  /// store_borrow'd value is created.  That begin_borrow is the instruction
+  /// after the store_borrow, and it is the other running value.
   BlockToInstMap initializationPoints;
 
   /// The first instruction in each block that deinitializes the storage that is
@@ -778,8 +782,9 @@ SILInstruction *StackAllocationPromoter::promoteAllocationInBlock(
   //                           it has been destroy_addr'd, etc
   // - Some + isStorageValid: a value was encountered and is currently stored
   Optional<StorageStateTracking<LiveValues>> runningVals;
-  // Keep track of the last StoreInst that we found and the BeginBorrowInst and
-  // CopyValueInst that we created in response if the alloc_stack was lexical.
+  // The most recent StoreInst or StoreBorrowInst that encountered while
+  // iterating over the block.  The final value will be returned to the caller
+  // which will use it to determine the live-out value of the block.
   SILInstruction *lastStoreInst = nullptr;
 
   // For all instructions in the block.
@@ -1002,15 +1007,8 @@ void StackAllocationPromoter::addBlockArguments(BlockSetVector &phiBlocks) {
   LLVM_DEBUG(llvm::dbgs() << "*** Adding new block arguments.\n");
 
   for (auto *block : phiBlocks) {
-    // The stored value.
+    // The stored value or its lexical move.
     block->createPhiArgument(asi->getElementType(), OwnershipKind::Owned);
-    if (lexicalLifetimeEnsured(asi)) {
-      // The borrow scope.
-      block->createPhiArgument(asi->getElementType(),
-                               OwnershipKind::Guaranteed);
-      // The copy of the borrowed value.
-      block->createPhiArgument(asi->getElementType(), OwnershipKind::Owned);
-    }
   }
 }
 
@@ -1033,7 +1031,7 @@ StackAllocationPromoter::getLiveOutValues(BlockSetVector &phiBlocks,
       LLVM_DEBUG(llvm::dbgs() << "*** Found Store def " << stored);
 
       if (!lexicalLifetimeEnsured(asi)) {
-        auto values = LiveValues::forOwned(stored, {}, {});
+        auto values = LiveValues::forOwned(stored, {});
         return values;
       }
       if (isa<StoreBorrowInst>(inst)) {
@@ -1045,30 +1043,19 @@ StackAllocationPromoter::getLiveOutValues(BlockSetVector &phiBlocks,
         auto values = LiveValues::forGuaranteed(stored, borrow);
         return values;
       }
-      auto borrow = cast<BeginBorrowInst>(inst->getNextInstruction());
-      auto copy = cast<CopyValueInst>(borrow->getNextInstruction());
-      auto values = LiveValues::forOwned(stored, borrow, copy);
+      auto move = cast<MoveValueInst>(inst->getNextInstruction());
+      auto values = LiveValues::forOwned(stored, move);
       return values;
     }
 
     // If there is a Phi definition in this block:
     if (phiBlocks.contains(domBlock)) {
       // Return the dummy instruction that represents the new value that we will
       // add to the basic block.
-      SILValue original;
-      SILValue borrow;
-      SILValue copy;
-      if (lexicalLifetimeEnsured(asi)) {
-        original = domBlock->getArgument(domBlock->getNumArguments() - 3);
-        borrow = domBlock->getArgument(domBlock->getNumArguments() - 2);
-        copy = domBlock->getArgument(domBlock->getNumArguments() - 1);
-      } else {
-        original = domBlock->getArgument(domBlock->getNumArguments() - 1);
-        borrow = SILValue();
-        copy = SILValue();
-      }
-      LLVM_DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *original);
-      auto values = LiveValues::forOwned(original, borrow, copy);
+      SILValue argument =
+          domBlock->getArgument(domBlock->getNumArguments() - 1);
+      LLVM_DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *argument);
+      auto values = LiveValues::toReplace(asi, argument);
       return values;
     }
 
@@ -1086,7 +1073,7 @@ StackAllocationPromoter::getEffectiveLiveOutValues(BlockSetVector &phiBlocks,
     return *values;
   }
   auto *undef = SILUndef::get(asi->getElementType(), *asi->getFunction());
-  return LiveValues::forOwned(undef, undef, undef);
+  return LiveValues::forOwned(undef, undef);
 }
 
 Optional<LiveValues>
@@ -1098,19 +1085,8 @@ StackAllocationPromoter::getLiveInValues(BlockSetVector &phiBlocks,
   // chain.
   if (phiBlocks.contains(block)) {
     LLVM_DEBUG(llvm::dbgs() << "*** Found a local Phi definition.\n");
-    SILValue original;
-    SILValue borrow;
-    SILValue copy;
-    if (lexicalLifetimeEnsured(asi)) {
-      original = block->getArgument(block->getNumArguments() - 3);
-      borrow = block->getArgument(block->getNumArguments() - 2);
-      copy = block->getArgument(block->getNumArguments() - 1);
-    } else {
-      original = block->getArgument(block->getNumArguments() - 1);
-      borrow = SILValue();
-      copy = SILValue();
-    }
-    auto values = LiveValues::forOwned(original, borrow, copy);
+    SILValue argument = block->getArgument(block->getNumArguments() - 1);
+    auto values = LiveValues::toReplace(asi, argument);
     return values;
   }
 
@@ -1134,7 +1110,7 @@ StackAllocationPromoter::getEffectiveLiveInValues(BlockSetVector &phiBlocks,
   }
   auto *undef = SILUndef::get(asi->getElementType(), *asi->getFunction());
   // TODO: Add another kind of LiveValues for undef.
-  return LiveValues::forOwned(undef, undef, undef);
+  return LiveValues::forOwned(undef, undef);
 }
 
 void StackAllocationPromoter::fixPhiPredBlock(BlockSetVector &phiBlocks,
@@ -1150,11 +1126,7 @@ void StackAllocationPromoter::fixPhiPredBlock(BlockSetVector &phiBlocks,
                           << ownedValues.stored);
 
   SmallVector<SILValue> vals;
-  vals.push_back(ownedValues.stored);
-  if (lexicalLifetimeEnsured(asi)) {
-    vals.push_back(ownedValues.borrow);
-    vals.push_back(ownedValues.copy);
-  }
+  vals.push_back(ownedValues.replacement(asi, nullptr));
 
   addArgumentsToBranch(vals, destBlock, ti);
   deleter.forceDelete(ti);
@@ -1318,10 +1290,6 @@ void StackAllocationPromoter::fixBranchesAndUses(BlockSetVector &phiBlocks,
           block->getArgument(block->getNumArguments() - 1));
       if (!livePhis.contains(proactivePhi)) {
         eraseLastPhiFromBlock(block);
-        if (lexicalLifetimeEnsured(asi)) {
-          eraseLastPhiFromBlock(block);
-          eraseLastPhiFromBlock(block);
-        }
       } else {
         phiBlocksOut.insert(block);
       }