Add capture checking to some standard library classes (#18192)

Convert some standard library classes to capture checking. The converted
classes are represented as tests.
Initially, the `Iterator` and `IterableOnce` classes are converted.
UPDATE: By now, we have quite a few more, including List, ListBuffer,
View, and all their super classes and traits.

Also change capture checker to facilitate the conversion. The main
changes are about better interop with classes that are not (yet) capture
checked.

Based on #18131

Author: Linyxus

compiler/src/dotty/tools/dotc/cc/CaptureOps.scala

@@ -15,6 +15,12 @@ import config.Feature
 private val Captures: Key[CaptureSet] = Key()
 private val BoxedType: Key[BoxedTypeCache] = Key()
 
+/** Switch whether unpickled function types and byname types should be mapped to
+ *  impure types. With the new gradual typing using Fluid capture sets, this should
+ *  be no longer needed. Also, it has bad interactions with pickling tests.
+ */
+private val adaptUnpickledFunctionTypes = false
+
 /** The arguments of a @retains or @retainsByName annotation */
 private[cc] def retainedElems(tree: Tree)(using Context): List[Tree] = tree match
   case Apply(_, Typed(SeqLiteral(elems, _), _) :: Nil) => elems
@@ -49,7 +55,7 @@ extension (tree: Tree)
    *  a by name parameter type, turning the latter into an impure by name parameter type.
    */
   def adaptByNameArgUnderPureFuns(using Context): Tree =
-    if Feature.pureFunsEnabledSomewhere then
+    if adaptUnpickledFunctionTypes && Feature.pureFunsEnabledSomewhere then
       val rbn = defn.RetainsByNameAnnot
       Annotated(tree,
         New(rbn.typeRef).select(rbn.primaryConstructor).appliedTo(
@@ -145,7 +151,7 @@ extension (tp: Type)
    */
   def adaptFunctionTypeUnderPureFuns(using Context): Type = tp match
     case AppliedType(fn, args)
-    if Feature.pureFunsEnabledSomewhere && defn.isFunctionClass(fn.typeSymbol) =>
+    if adaptUnpickledFunctionTypes && Feature.pureFunsEnabledSomewhere && defn.isFunctionClass(fn.typeSymbol) =>
       val fname = fn.typeSymbol.name
       defn.FunctionType(
         fname.functionArity,

compiler/src/dotty/tools/dotc/cc/CaptureSet.scala

@@ -356,6 +356,20 @@ object CaptureSet:
     override def toString = elems.toString
   end Const
 
+  /** A special capture set that gets added to the types of symbols that were not
+   *  themselves capture checked, in order to admit arbitrary corresponding capture
+   *  sets in subcapturing comparisons. Similar to platform types for explicit
+   *  nulls, this provides more lenient checking against compilation units that
+   *  were not yet compiled with capture checking on.
+   */
+  object Fluid extends Const(emptySet):
+    override def isAlwaysEmpty = false
+    override def addNewElems(elems: Refs, origin: CaptureSet)(using Context, VarState) = CompareResult.OK
+    override def accountsFor(x: CaptureRef)(using Context): Boolean = true
+    override def mightAccountFor(x: CaptureRef)(using Context): Boolean = true
+    override def toString = "<fluid>"
+  end Fluid
+
   /** The subclass of captureset variables with given initial elements */
   class Var(initialElems: Refs = emptySet) extends CaptureSet:
 
@@ -863,7 +877,7 @@ object CaptureSet:
         case CapturingType(parent, refs) =>
           recur(parent) ++ refs
         case tpd @ RefinedType(parent, _, rinfo: MethodType)
-        if followResult && defn.isFunctionType(tpd) =>
+        if followResult && defn.isFunctionNType(tpd) =>
           ofType(parent, followResult = false)                 // pick up capture set from parent type
           ++ (recur(rinfo.resType)                             // add capture set of result
           -- CaptureSet(rinfo.paramRefs.filter(_.isTracked)*)) // but disregard bound parameters

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -290,14 +290,46 @@ class CheckCaptures extends Recheck, SymTransformer:
     def includeCallCaptures(sym: Symbol, pos: SrcPos)(using Context): Unit =
       if sym.exists && curEnv.isOpen then markFree(capturedVars(sym), pos)
 
+    private def handleBackwardsCompat(tp: Type, sym: Symbol, initialVariance: Int = 1)(using Context): Type =
+      val fluidify = new TypeMap with IdempotentCaptRefMap:
+        variance = initialVariance
+        def apply(t: Type): Type = t match
+          case t: MethodType =>
+            mapOver(t)
+          case t: TypeLambda =>
+            t.derivedLambdaType(resType = this(t.resType))
+          case CapturingType(_, _) =>
+            t
+          case _ =>
+            val t1  = t match
+              case t @ RefinedType(parent, rname, rinfo: MethodType) if defn.isFunctionType(t) =>
+                t.derivedRefinedType(parent, rname, this(rinfo))
+              case _ =>
+                mapOver(t)
+            if variance > 0 then t1
+            else Setup.decorate(t1, Function.const(CaptureSet.Fluid))
+
+      def isPreCC(sym: Symbol): Boolean =
+        sym.isTerm && sym.maybeOwner.isClass
+        && !sym.owner.is(CaptureChecked)
+        && !defn.isFunctionSymbol(sym.owner)
+
+      if isPreCC(sym) then
+        val tpw = tp.widen
+        val fluidTp = fluidify(tpw)
+        if fluidTp eq tpw then tp
+        else fluidTp.showing(i"fluid for ${sym.showLocated}, ${sym.is(JavaDefined)}: $tp --> $result", capt)
+      else tp
+    end handleBackwardsCompat
+
     override def recheckIdent(tree: Ident)(using Context): Type =
       if tree.symbol.is(Method) then
         if tree.symbol.info.isParameterless then
           // there won't be an apply; need to include call captures now
           includeCallCaptures(tree.symbol, tree.srcPos)
       else
         markFree(tree.symbol, tree.srcPos)
-      super.recheckIdent(tree)
+      handleBackwardsCompat(super.recheckIdent(tree), tree.symbol)
 
     /** A specialized implementation of the selection rule.
      *
@@ -327,7 +359,7 @@ class CheckCaptures extends Recheck, SymTransformer:
       val selType = recheckSelection(tree, qualType, name, disambiguate)
       val selCs = selType.widen.captureSet
       if selCs.isAlwaysEmpty || selType.widen.isBoxedCapturing || qualType.isBoxedCapturing then
-        selType
+        handleBackwardsCompat(selType, tree.symbol)
       else
         val qualCs = qualType.captureSet
         capt.println(i"pick one of $qualType, ${selType.widen}, $qualCs, $selCs in $tree")
@@ -362,7 +394,16 @@ class CheckCaptures extends Recheck, SymTransformer:
         val argType0 = f(recheckStart(arg, pt))
         val argType = super.recheckFinish(argType0, arg, pt)
         super.recheckFinish(argType, tree, pt)
-      if meth == defn.Caps_unsafeBox then
+
+      if meth == defn.Caps_unsafeAssumePure then
+        val arg :: Nil = tree.args: @unchecked
+        val argType0 = recheck(arg, pt.capturing(CaptureSet.universal))
+        val argType =
+          if argType0.captureSet.isAlwaysEmpty then argType0
+          else argType0.widen.stripCapturing
+        capt.println(i"rechecking $arg with ${pt.capturing(CaptureSet.universal)}: $argType")
+        super.recheckFinish(argType, tree, pt)
+      else if meth == defn.Caps_unsafeBox then
         mapArgUsing(_.forceBoxStatus(true))
       else if meth == defn.Caps_unsafeUnbox then
         mapArgUsing(_.forceBoxStatus(false))
@@ -662,8 +703,10 @@ class CheckCaptures extends Recheck, SymTransformer:
     /** Turn `expected` into a dependent function when `actual` is dependent. */
     private def alignDependentFunction(expected: Type, actual: Type)(using Context): Type =
       def recur(expected: Type): Type = expected.dealias match
-        case expected @ CapturingType(eparent, refs) =>
-          CapturingType(recur(eparent), refs, boxed = expected.isBoxed)
+        case expected0 @ CapturingType(eparent, refs) =>
+          val eparent1 = recur(eparent)
+          if eparent1 eq eparent then expected
+          else CapturingType(eparent1, refs, boxed = expected0.isBoxed)
         case expected @ defn.FunctionOf(args, resultType, isContextual)
           if defn.isNonRefinedFunction(expected) && defn.isFunctionNType(actual) && !defn.isNonRefinedFunction(actual) =>
           val expected1 = toDepFun(args, resultType, isContextual)
@@ -883,7 +926,7 @@ class CheckCaptures extends Recheck, SymTransformer:
     *  But maybe we can then elide the check during the RefChecks phase under captureChecking?
     */
     def checkOverrides = new TreeTraverser:
-      class OverridingPairsCheckerCC(clazz: ClassSymbol, self: Type, srcPos: SrcPos)(using Context) extends OverridingPairsChecker(clazz, self) {
+      class OverridingPairsCheckerCC(clazz: ClassSymbol, self: Type, srcPos: SrcPos)(using Context) extends OverridingPairsChecker(clazz, self):
         /** Check subtype with box adaptation.
         *  This function is passed to RefChecks to check the compatibility of overriding pairs.
         *  @param sym  symbol of the field definition that is being checked
@@ -905,7 +948,11 @@ class CheckCaptures extends Recheck, SymTransformer:
                 case _ => adapted
             finally curEnv = saved
           actual1 frozen_<:< expected1
-      }
+
+        override def adjustInfo(tp: Type, member: Symbol)(using Context): Type =
+          handleBackwardsCompat(tp, member, initialVariance = 0)
+            //.showing(i"adjust $other: $tp --> $result")
+      end OverridingPairsCheckerCC
 
       def traverse(t: Tree)(using Context) =
         t match

compiler/src/dotty/tools/dotc/cc/Setup.scala

@@ -114,88 +114,6 @@ extends tpd.TreeTraverser:
           case _ => tp
       case _ => tp
 
-    private def superTypeIsImpure(tp: Type): Boolean = {
-      tp.dealias match
-        case CapturingType(_, refs) =>
-          !refs.isAlwaysEmpty
-        case tp: (TypeRef | AppliedType) =>
-          val sym = tp.typeSymbol
-          if sym.isClass then
-            sym == defn.AnyClass
-              // we assume Any is a shorthand of {cap} Any, so if Any is an upper
-              // bound, the type is taken to be impure.
-          else superTypeIsImpure(tp.superType)
-        case tp: (RefinedOrRecType | MatchType) =>
-          superTypeIsImpure(tp.underlying)
-        case tp: AndType =>
-          superTypeIsImpure(tp.tp1) || needsVariable(tp.tp2)
-        case tp: OrType =>
-          superTypeIsImpure(tp.tp1) && superTypeIsImpure(tp.tp2)
-        case _ =>
-          false
-    }.showing(i"super type is impure $tp = $result", capt)
-
-    /** Should a capture set variable be added on type `tp`? */
-    def needsVariable(tp: Type): Boolean = {
-      tp.typeParams.isEmpty && tp.match
-        case tp: (TypeRef | AppliedType) =>
-          val tp1 = tp.dealias
-          if tp1 ne tp then needsVariable(tp1)
-          else
-            val sym = tp1.typeSymbol
-            if sym.isClass then
-              !sym.isPureClass && sym != defn.AnyClass
-            else superTypeIsImpure(tp1)
-        case tp: (RefinedOrRecType | MatchType) =>
-          needsVariable(tp.underlying)
-        case tp: AndType =>
-          needsVariable(tp.tp1) && needsVariable(tp.tp2)
-        case tp: OrType =>
-          needsVariable(tp.tp1) || needsVariable(tp.tp2)
-        case CapturingType(parent, refs) =>
-          needsVariable(parent)
-          && refs.isConst      // if refs is a variable, no need to add another
-          && !refs.isUniversal // if refs is {cap}, an added variable would not change anything
-        case _ =>
-          false
-    }.showing(i"can have inferred capture $tp = $result", capt)
-
-    /** Add a capture set variable to `tp` if necessary, or maybe pull out
-     *  an embedded capture set variable from a part of `tp`.
-     */
-    def addVar(tp: Type) = tp match
-      case tp @ RefinedType(parent @ CapturingType(parent1, refs), rname, rinfo) =>
-        CapturingType(tp.derivedRefinedType(parent1, rname, rinfo), refs, parent.isBoxed)
-      case tp: RecType =>
-        tp.parent match
-          case parent @ CapturingType(parent1, refs) =>
-            CapturingType(tp.derivedRecType(parent1), refs, parent.isBoxed)
-          case _ =>
-            tp // can return `tp` here since unlike RefinedTypes, RecTypes are never created
-                // by `mapInferred`. Hence if the underlying type admits capture variables
-                // a variable was already added, and the first case above would apply.
-      case AndType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
-        assert(refs1.asVar.elems.isEmpty)
-        assert(refs2.asVar.elems.isEmpty)
-        assert(tp1.isBoxed == tp2.isBoxed)
-        CapturingType(AndType(parent1, parent2), refs1 ** refs2, tp1.isBoxed)
-      case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
-        assert(refs1.asVar.elems.isEmpty)
-        assert(refs2.asVar.elems.isEmpty)
-        assert(tp1.isBoxed == tp2.isBoxed)
-        CapturingType(OrType(parent1, parent2, tp.isSoft), refs1 ++ refs2, tp1.isBoxed)
-      case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2) =>
-        CapturingType(OrType(parent1, tp2, tp.isSoft), refs1, tp1.isBoxed)
-      case tp @ OrType(tp1, tp2 @ CapturingType(parent2, refs2)) =>
-        CapturingType(OrType(tp1, parent2, tp.isSoft), refs2, tp2.isBoxed)
-      case _ if needsVariable(tp) =>
-        val cs = tp.dealias match
-          case CapturingType(_, refs) => CaptureSet.Var(refs.elems)
-          case _ => CaptureSet.Var()
-        CapturingType(tp, cs)
-      case _ =>
-        tp
-
     private var isTopLevel = true
 
     private def mapNested(ts: List[Type]): List[Type] =
@@ -246,7 +164,7 @@ extends tpd.TreeTraverser:
             resType = this(tp.resType))
         case _ =>
           mapOver(tp)
-      addVar(addCaptureRefinements(tp1))
+      Setup.addVar(addCaptureRefinements(tp1))
     end apply
   end mapInferred
 
@@ -385,9 +303,9 @@ extends tpd.TreeTraverser:
           val polyType = fn.tpe.widen.asInstanceOf[TypeLambda]
           for case (arg: TypeTree, pinfo, pname) <- args.lazyZip(polyType.paramInfos).lazyZip((polyType.paramNames)) do
             if pinfo.bounds.hi.hasAnnotation(defn.Caps_SealedAnnot) then
-              def where = if fn.symbol.exists then i" in the body of ${fn.symbol}" else ""
+              def where = if fn.symbol.exists then i" in an argument of ${fn.symbol}" else ""
               CheckCaptures.disallowRootCapabilitiesIn(arg.knownType,
-                i"Sealed type variable $pname", " be instantiated to",
+                i"Sealed type variable $pname", "be instantiated to",
                 i"This is often caused by a local capability$where\nleaking as part of its result.",
                 tree.srcPos)
       case _ =>
@@ -428,7 +346,7 @@ extends tpd.TreeTraverser:
               if prevLambdas.isEmpty then restp
               else SubstParams(prevPsymss, prevLambdas)(restp)
 
-        if tree.tpt.hasRememberedType && !sym.isConstructor then
+        if sym.exists && tree.tpt.hasRememberedType && !sym.isConstructor then
           val newInfo = integrateRT(sym.info, sym.paramSymss, Nil, Nil)
             .showing(i"update info $sym: ${sym.info} --> $result", capt)
           if newInfo ne sym.info then
@@ -474,4 +392,97 @@ object Setup:
 
   def isDuringSetup(using Context): Boolean =
     ctx.property(IsDuringSetupKey).isDefined
+
+  private def superTypeIsImpure(tp: Type)(using Context): Boolean = {
+    tp.dealias match
+      case CapturingType(_, refs) =>
+        !refs.isAlwaysEmpty
+      case tp: (TypeRef | AppliedType) =>
+        val sym = tp.typeSymbol
+        if sym.isClass then
+          sym == defn.AnyClass
+            // we assume Any is a shorthand of {cap} Any, so if Any is an upper
+            // bound, the type is taken to be impure.
+        else superTypeIsImpure(tp.superType)
+      case tp: (RefinedOrRecType | MatchType) =>
+        superTypeIsImpure(tp.underlying)
+      case tp: AndType =>
+        superTypeIsImpure(tp.tp1) || needsVariable(tp.tp2)
+      case tp: OrType =>
+        superTypeIsImpure(tp.tp1) && superTypeIsImpure(tp.tp2)
+      case _ =>
+        false
+  }.showing(i"super type is impure $tp = $result", capt)
+
+  /** Should a capture set variable be added on type `tp`? */
+  def needsVariable(tp: Type)(using Context): Boolean = {
+    tp.typeParams.isEmpty && tp.match
+      case tp: (TypeRef | AppliedType) =>
+        val sym = tp.typeSymbol
+        if sym.isClass then
+          !sym.isPureClass && sym != defn.AnyClass
+        else
+          sym != defn.FromJavaObjectSymbol
+            // For capture checking, we assume Object from Java is the same as Any
+          && {
+            val tp1 = tp.dealias
+            if tp1 ne tp then needsVariable(tp1)
+            else superTypeIsImpure(tp1)
+          }
+      case tp: (RefinedOrRecType | MatchType) =>
+        needsVariable(tp.underlying)
+      case tp: AndType =>
+        needsVariable(tp.tp1) && needsVariable(tp.tp2)
+      case tp: OrType =>
+        needsVariable(tp.tp1) || needsVariable(tp.tp2)
+      case CapturingType(parent, refs) =>
+        needsVariable(parent)
+        && refs.isConst      // if refs is a variable, no need to add another
+        && !refs.isUniversal // if refs is {cap}, an added variable would not change anything
+      case _ =>
+        false
+  }.showing(i"can have inferred capture $tp = $result", capt)
+
+  /** Add a capture set variable to `tp` if necessary, or maybe pull out
+   *  an embedded capture set variable from a part of `tp`.
+   */
+  def decorate(tp: Type, addedSet: Type => CaptureSet)(using Context): Type = tp match
+    case tp @ RefinedType(parent @ CapturingType(parent1, refs), rname, rinfo) =>
+      CapturingType(tp.derivedRefinedType(parent1, rname, rinfo), refs, parent.isBoxed)
+    case tp: RecType =>
+      tp.parent match
+        case parent @ CapturingType(parent1, refs) =>
+          CapturingType(tp.derivedRecType(parent1), refs, parent.isBoxed)
+        case _ =>
+          tp // can return `tp` here since unlike RefinedTypes, RecTypes are never created
+              // by `mapInferred`. Hence if the underlying type admits capture variables
+              // a variable was already added, and the first case above would apply.
+    case AndType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
+      assert(refs1.elems.isEmpty)
+      assert(refs2.elems.isEmpty)
+      assert(tp1.isBoxed == tp2.isBoxed)
+      CapturingType(AndType(parent1, parent2), refs1 ** refs2, tp1.isBoxed)
+    case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
+      assert(refs1.elems.isEmpty)
+      assert(refs2.elems.isEmpty)
+      assert(tp1.isBoxed == tp2.isBoxed)
+      CapturingType(OrType(parent1, parent2, tp.isSoft), refs1 ++ refs2, tp1.isBoxed)
+    case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2) =>
+      CapturingType(OrType(parent1, tp2, tp.isSoft), refs1, tp1.isBoxed)
+    case tp @ OrType(tp1, tp2 @ CapturingType(parent2, refs2)) =>
+      CapturingType(OrType(tp1, parent2, tp.isSoft), refs2, tp2.isBoxed)
+    case _ if needsVariable(tp) =>
+      CapturingType(tp, addedSet(tp))
+    case _ =>
+      tp
+
+  /** Add a capture set variable to `tp` if necessary, or maybe pull out
+   *  an embedded capture set variable from a part of `tp`.
+   */
+  def addVar(tp: Type)(using Context): Type =
+    decorate(tp,
+      addedSet = _.dealias.match
+        case CapturingType(_, refs) => CaptureSet.Var(refs.elems)
+        case _ => CaptureSet.Var())
+
 end Setup