Merge pull request #60670 from nate-chandler/lexical_lifetimes/owned_arguments

[SIL] Maintain owned argument lifetimes at inlining.

Author: nate-chandler

SwiftCompilerSources/Sources/Optimizer/Utilities/WalkUtils.swift

@@ -341,7 +341,7 @@ extension ValueDefUseWalker {
         return unmatchedPath(value: operand, path: path)
       }
     case is InitExistentialRefInst, is OpenExistentialRefInst,
-      is BeginBorrowInst, is CopyValueInst,
+      is BeginBorrowInst, is CopyValueInst, is MoveValueInst,
       is UpcastInst, is UncheckedRefCastInst, is EndCOWMutationInst,
       is RefToBridgeObjectInst, is BridgeObjectToRefInst, is MarkMustCheckInst:
       return walkDownUses(ofValue: (instruction as! SingleValueInstruction), path: path)
@@ -618,7 +618,7 @@ extension ValueUseDefWalker {
         return rootDef(value: mvr, path: path)
       }
     case is InitExistentialRefInst, is OpenExistentialRefInst,
-      is BeginBorrowInst, is CopyValueInst,
+      is BeginBorrowInst, is CopyValueInst, is MoveValueInst,
       is UpcastInst, is UncheckedRefCastInst, is EndCOWMutationInst,
       is RefToBridgeObjectInst, is BridgeObjectToRefInst, is MarkMustCheckInst:
       return walkUp(value: (def as! Instruction).operands[0].value, path: path)

SwiftCompilerSources/Sources/SIL/Instruction.swift

@@ -609,6 +609,8 @@ final public class ProjectBoxInst : SingleValueInstruction, UnaryInstruction {
 
 final public class CopyValueInst : SingleValueInstruction, UnaryInstruction {}
 
+final public class MoveValueInst : SingleValueInstruction, UnaryInstruction {}
+
 final public class StrongCopyUnownedValueInst : SingleValueInstruction, UnaryInstruction {}
 
 final public class StrongCopyUnmanagedValueInst : SingleValueInstruction, UnaryInstruction  {}

SwiftCompilerSources/Sources/SIL/Registration.swift

@@ -123,6 +123,7 @@ public func registerSILClasses() {
   register(BeginBorrowInst.self)
   register(ProjectBoxInst.self)
   register(CopyValueInst.self)
+  register(MoveValueInst.self)
   register(EndCOWMutationInst.self)
   register(ClassifyBridgeObjectInst.self)
   register(PartialApplyInst.self)

docs/SIL.rst

@@ -2414,27 +2414,55 @@ Variable Lifetimes
 In order for programmer intended lifetimes to be maintained under optimization,
 the lifetimes of SIL values which correspond to named source-level values can
 only be modified in limited ways.  Generally, the behavior is that the lifetime
-of a named source-level value cannot _observably_ end before the end of the
-lexical scope in which that value is defined.  Specifically, code motion may
-not move the ends of these lifetimes across a **deinit barrier**.
-
-A few sorts of SIL value have lifetimes that are constrained that way:
+of a named source-level value is anchored to the variable's lexical scope and
+confined by **deinit barriers**.  Specifically, code motion may not move the
+ends of these lifetimes across a deinit barrier.
+
+Source level variables (lets, vars, ...) and function arguments will result in
+SIL-level lexical lifetimes if either of the two sets of circumstances apply:
+(1) Inferred lexicality.
+- the type is non-trivial
+- the type is not eager-move
+- the variable or argument is not annotated to be eager-move
+OR
+(2) Explicit lexicality.
+- the type, variable, or argument is annotated `@_lexical`
+
+A type is eager-move by satisfying one of two conditions:
+(1) Inferred: An aggregate is inferred to be eager-move if all of its fields are
+    eager-move.
+(2) Annotated: Any type can be eager-move if it is annotated with an attribute
+    that explicitly specifies it to be: `@_eagerMove`, `@_noImplicitCopy`.
+
+A variable or argument is eager-move by satisfying one of two conditions:
+(1) Inferred: Its type is eager-move.
+(2) Annotated: The variable or argument is annotated with an attribute that
+    specifies it to be: `@_eagerMove`, `@_noImplicitCopy`.
+
+These source-level rules result in a few sorts of SIL value whose destroys must
+not be moved across deinit barriers:
 
 1: `begin_borrow [lexical]`
 2: `move_value [lexical]`
-3: @owned function arguments
+3: function arguments
 4: `alloc_stack [lexical]`
 
-That these three have constrained lifetimes is encoded in ValueBase::isLexical,
-which should be checked before changing the lifetime of a value.
+To translate from the source-level representation of lexicality to the
+SIL-level representation, for source-level variables (vars, lets, ...) SILGen
+generates `begin_borrow [lexical]`, `move_value [lexical]`, `alloc_stack
+[lexical]` .  For function arguments, there is no work to do:
+a `SILFunctionArgument` itself can be lexical
+(`SILFunctionArgument::isLexical`).
+
+That the first three have constrained lifetimes is encoded in
+ValueBase::isLexical, which should be checked before changing the lifetime of a
+value.
 
-The reason that only @owned function arguments are constrained is that a
-@guaranteed function argument is guaranteed by the function's caller to live for
-the full duration of the function already.  Optimization of the function alone
-can't shorten it.  When such a function is inlined into its caller, though, a
-lexical borrow scope is added for each of its @guaranteed arguments, ensuring
-that the lifetime of the corresponding source-level value is not shortened in a
-way that doesn't respect deinit barriers.
+When a function is inlined into its caller, a lexical borrow scope is added for
+each of its @guaranteed arguments, and a lexical move is added for each of its
+@owned arguments, (unless the values being passed are already lexical
+themselves) ensuring that the lifetimes of the corresponding source-level
+values are not shortened in a way that doesn't respect deinit barriers.
 
 Unlike the other sorts, `alloc_stack [lexical]` isn't a SILValue.  Instead, it
 constrains the lifetime of an addressable variable.  Since the constraint is

include/swift/SIL/InstructionUtils.h

@@ -88,8 +88,8 @@ SILValue stripBorrow(SILValue V);
 //===----------------------------------------------------------------------===//
 
 /// Return a non-null SingleValueInstruction if the given instruction merely
-/// copies the value of its first operand, possibly changing its type or
-/// ownership state, but otherwise having no effect.
+/// copies or moves the value of its first operand, possibly changing its type
+/// or ownership state, but otherwise having no effect.
 ///
 /// The returned instruction may have additional "incidental" operands;
 /// mark_dependence for example.

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -109,7 +109,7 @@
 
 namespace swift {
 
-extern llvm::Statistic NumCopiesEliminated;
+extern llvm::Statistic NumCopiesAndMovesEliminated;
 extern llvm::Statistic NumCopiesGenerated;
 
 /// Insert a copy on this operand. Trace and update stats.

lib/SIL/IR/SILValue.cpp

@@ -116,11 +116,8 @@ ValueBase::getDefiningInstructionResult() {
 }
 
 bool ValueBase::isLexical() const {
-  if (auto *argument = dyn_cast<SILFunctionArgument>(this)) {
-    // TODO: Recognize guaranteed arguments as lexical too.
-    return argument->getOwnershipKind() == OwnershipKind::Owned &&
-           argument->getLifetime().isLexical();
-  }
+  if (auto *argument = dyn_cast<SILFunctionArgument>(this))
+    return argument->getLifetime().isLexical();
   if (auto *bbi = dyn_cast<BeginBorrowInst>(this))
     return bbi->isLexical();
   if (auto *mvi = dyn_cast<MoveValueInst>(this))

lib/SIL/Utils/InstructionUtils.cpp

@@ -31,6 +31,7 @@ SILValue swift::lookThroughOwnershipInsts(SILValue v) {
     switch (v->getKind()) {
     default:
       return v;
+    case ValueKind::MoveValueInst:
     case ValueKind::CopyValueInst:
     case ValueKind::BeginBorrowInst:
       v = cast<SingleValueInstruction>(v)->getOperand(0);
@@ -255,6 +256,7 @@ SingleValueInstruction *swift::getSingleValueCopyOrCast(SILInstruction *I) {
   case SILInstructionKind::BeginBorrowInst:
   case SILInstructionKind::BeginAccessInst:
   case SILInstructionKind::MarkDependenceInst:
+  case SILInstructionKind::MoveValueInst:
     return cast<SingleValueInstruction>(I);
   }
 }

lib/SIL/Utils/OwnershipUtils.cpp

@@ -2414,16 +2414,25 @@ void swift::visitTransitiveEndBorrows(
 ///
 /// A begin_borrow [lexical] is nested if the borrowed value's lifetime is
 /// guaranteed by another lexical scope.  That happens if:
-/// - the value is a guaranteed argument to the function
-/// - the value is itself a begin_borrow [lexical]
+/// - the non-guaranteed borrowee's value is lexical
+/// - the guaranteed borrowee's value's reference roots are lexical
+///   - for example, the borrowee is itself a begin_borrow [lexical]
 bool swift::isNestedLexicalBeginBorrow(BeginBorrowInst *bbi) {
   assert(bbi->isLexical());
   auto value = bbi->getOperand();
-  if (auto *outerBBI = dyn_cast<BeginBorrowInst>(value)) {
-    return outerBBI->isLexical();
+  if (value->getOwnershipKind() != OwnershipKind::Guaranteed) {
+    return value->isLexical();
   }
-  if (auto *arg = dyn_cast<SILFunctionArgument>(value)) {
-    return arg->getOwnershipKind() == OwnershipKind::Guaranteed;
-  }
-  return false;
+  SmallVector<SILValue, 8> roots;
+  findGuaranteedReferenceRoots(value, /*lookThroughNestedBorrows=*/false,
+                               roots);
+  return llvm::all_of(roots, [](auto root) {
+    if (auto *outerBBI = dyn_cast<BeginBorrowInst>(root)) {
+      return outerBBI->isLexical();
+    }
+    if (auto *arg = dyn_cast<SILFunctionArgument>(root)) {
+      return arg->getOwnershipKind() == OwnershipKind::Guaranteed;
+    }
+    return false;
+  });
 }

lib/SIL/Utils/Projection.cpp

@@ -387,7 +387,7 @@ Optional<ProjectionPath> ProjectionPath::getProjectionPath(SILValue Start,
     // TODO: migrate users to getProjectionPath to the AccessPath utility to
     // avoid this hack.
     if (!isa<EndCOWMutationInst>(Iter) && !isa<BeginAccessInst>(Iter) &&
-        !isa<BeginBorrowInst>(Iter)) {
+        !isa<BeginBorrowInst>(Iter) && !isa<MoveValueInst>(Iter)) {
       Projection AP(Iter);
       if (!AP.isValid())
         break;