Fix special capture set handling in recheckApply, Step 1

Step1: refactor

The logic was querying the original types of trees, but we want the rechecked types instead.

Author: odersky

compiler/src/dotty/tools/dotc/cc/CaptureOps.scala

@@ -274,8 +274,8 @@ extension (tp: Type)
     case _ =>
       tp
 
-  /** Is type known to be always pure by its class structure,
-   *  so that adding a capture set to it would not make sense?
+  /** Is type known to be always pure by its class structure?
+   *  In that case, adding a capture set to it would not make sense.
    */
   def isAlwaysPure(using Context): Boolean = tp.dealias match
     case tp: (TypeRef | AppliedType) =>

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -463,8 +463,8 @@ class CheckCaptures extends Recheck, SymTransformer:
 
     /** A specialized implementation of the selection rule.
      *
-     *  E |- f: f{ m: Cr R }^Cf
-     *  -----------------------
+     *  E |- f: T{ m: R^Cr }^{f}
+     *  ------------------------
      *  E |- f.m: R^C
      *
      *  The implementation picks as `C` one of `{f}` or `Cr`, depending on the
@@ -507,17 +507,6 @@ class CheckCaptures extends Recheck, SymTransformer:
           selType
     }//.showing(i"recheck sel $tree, $qualType = $result")
 
-    /** A specialized implementation of the apply rule.
-     *
-     *  E |- f: Ra ->Cf Rr^Cr
-     *  E |- a: Ra^Ca
-     *  ---------------------
-     *  E |- f a: Rr^C
-     *
-     *  The implementation picks as `C` one of `{f, a}` or `Cr`, depending on the
-     *  outcome of a `mightSubcapture` test. It picks `{f, a}` if this might subcapture Cr
-     *  and Cr otherwise.
-     */
     override def recheckApply(tree: Apply, pt: Type)(using Context): Type =
       val meth = tree.fun.symbol
 
@@ -552,31 +541,47 @@ class CheckCaptures extends Recheck, SymTransformer:
             tp.derivedCapturingType(forceBox(parent), refs)
         mapArgUsing(forceBox)
       else
-        handleCall(meth, tree, () => Existential.toCap(super.recheckApply(tree, pt))) match
-          case appType @ CapturingType(appType1, refs) =>
-            tree.fun match
-              case Select(qual, _)
-              if !tree.fun.symbol.isConstructor
-                  && !qual.tpe.isBoxedCapturing
-                  && !tree.args.exists(_.tpe.isBoxedCapturing)
-                  && qual.tpe.captureSet.mightSubcapture(refs)
-                  && tree.args.forall(_.tpe.captureSet.mightSubcapture(refs))
-              =>
-                val callCaptures = tree.args.foldLeft(qual.tpe.captureSet): (cs, arg) =>
-                  cs ++ arg.tpe.captureSet
-                appType.derivedCapturingType(appType1, callCaptures)
-                  .showing(i"narrow $tree: $appType, refs = $refs, qual-cs = ${qual.tpe.captureSet} = $result", capt)
-              case _ => appType
-          case appType => appType
+        handleCall(meth, tree, () => super.recheckApply(tree, pt))
     end recheckApply
 
-    override def recheckArg(arg: Tree, formal: Type)(using Context): Type =
+    protected override
+    def recheckArg(arg: Tree, formal: Type)(using Context): Type =
       val argType = recheck(arg, formal)
       if unboxedArgs.remove(arg) && ccConfig.useUnboxedParams then
         capt.println(i"charging deep capture set of $arg: ${argType} = ${CaptureSet.deepCaptureSet(argType)}")
         markFree(CaptureSet.deepCaptureSet(argType), arg.srcPos)
       argType
 
+    /** A specialized implementation of the apply rule.
+     *
+     *  E |- f: Ra ->Cf Rr^Cr
+     *  E |- a: Ra^Ca
+     *  ---------------------
+     *  E |- f a: Rr^C
+     *
+     *  The implementation picks as `C` one of `{f, a}` or `Cr`, depending on the
+     *  outcome of a `mightSubcapture` test. It picks `{f, a}` if this might subcapture Cr
+     *  and Cr otherwise.
+     */
+    protected override
+    def recheckApplication(tree: Apply, qualType: Type, funType: MethodType, argTypes: List[Type])(using Context): Type =
+      Existential.toCap(super.recheckApplication(tree, qualType, funType, argTypes)) match
+        case appType @ CapturingType(appType1, refs)
+        if qualType.exists
+            && !tree.fun.symbol.isConstructor
+            && !qualType.isBoxedCapturing // TODO: This is not strng enough, we also have
+                 // to exclude existentials in function results
+            && !argTypes.exists(_.isBoxedCapturing)
+            && qualType.captureSet.mightSubcapture(refs)
+            && argTypes.forall(_.captureSet.mightSubcapture(refs))
+        =>
+          val callCaptures = tree.args.foldLeft(qualType.captureSet): (cs, arg) =>
+              cs ++ arg.tpe.captureSet
+          appType.derivedCapturingType(appType1, callCaptures)
+            .showing(i"narrow $tree: $appType, refs = $refs, qual-cs = ${qualType.captureSet} = $result", capt)
+        case appType =>
+          appType
+
     private def isDistinct(xs: List[Type]): Boolean = xs match
       case x :: xs1 => xs1.isEmpty || !xs1.contains(x) && isDistinct(xs1)
       case Nil => true

compiler/src/dotty/tools/dotc/transform/Recheck.scala

@@ -202,11 +202,13 @@ abstract class Recheck extends Phase, SymTransformer:
       tree.tpe
 
     def recheckSelect(tree: Select, pt: Type)(using Context): Type =
-      val Select(qual, name) = tree
+      recheckSelection(tree, recheckSelectQualifier(tree), tree.name, pt)
+
+    def recheckSelectQualifier(tree: Select)(using Context): Type =
       val proto =
         if tree.symbol == defn.Any_asInstanceOf then WildcardType
         else AnySelectionProto
-      recheckSelection(tree, recheck(qual, proto).widenIfUnstable, name, pt)
+      recheck(tree.qualifier, proto).widenIfUnstable
 
     def recheckSelection(tree: Select, qualType: Type, name: Name,
         sharpen: Denotation => Denotation)(using Context): Type =
@@ -292,8 +294,23 @@ abstract class Recheck extends Phase, SymTransformer:
     protected def recheckArg(arg: Tree, formal: Type)(using Context): Type =
       recheck(arg, formal)
 
+    /** A hook to check all the parts of an application:
+     *   @param  tree      the application `fn(args)`
+     *   @param  qualType  if the `fn` is a select `q.m`, the type of the qualifier `q`,
+     *                     otherwise NoType
+     *   @param  funType   the method type of `fn`
+     *   @param  argTypes  the types of the arguments
+     */
+    protected def recheckApplication(tree: Apply, qualType: Type, funType: MethodType, argTypes: List[Type])(using Context): Type =
+      constFold(tree, instantiate(funType, argTypes, tree.fun.symbol))
+
     def recheckApply(tree: Apply, pt: Type)(using Context): Type =
-      val funtpe0 = recheck(tree.fun)
+      val (funtpe0, qualType) = tree.fun match
+        case fun: Select =>
+          val qualType = recheckSelectQualifier(fun)
+          (recheckSelection(fun, qualType, fun.name, WildcardType), qualType)
+        case _ =>
+          (recheck(tree.fun), NoType)
       // reuse the tree's type on signature polymorphic methods, instead of using the (wrong) rechecked one
       val funtpe1 = if tree.fun.symbol.originalSignaturePolymorphic.exists then tree.fun.tpe else funtpe0
       funtpe1.widen match
@@ -316,7 +333,7 @@ abstract class Recheck extends Phase, SymTransformer:
               assert(formals.isEmpty)
               Nil
           val argTypes = recheckArgs(tree.args, formals, fntpe.paramRefs)
-          constFold(tree, instantiate(fntpe, argTypes, tree.fun.symbol))
+          recheckApplication(tree, qualType, fntpe1, argTypes)
             //.showing(i"typed app $tree : $fntpe with ${tree.args}%, % : $argTypes%, % = $result")
         case tp =>
           assert(false, i"unexpected type of ${tree.fun}: $tp")

tests/neg-custom-args/captures/caseclass/Test_2.scala

@@ -22,4 +22,4 @@ def test(c: C) =
 
   val y4 = y3 match
     case Ref(xx) => xx
-  val y4c: () ->{x3} Unit = y4
+  val y4c: () ->{y3} Unit = y4

tests/neg-custom-args/captures/reaches.check

@@ -34,6 +34,16 @@
    |                               that type captures the root capability `cap`.
    |                               This is often caused by a local capability in an argument of constructor Id
    |                               leaking as part of its result.
+-- [E007] Type Mismatch Error: tests/neg-custom-args/captures/reaches.scala:57:6 ---------------------------------------
+57 |    id(() => f.write())  // error
+   |    ^^^^^^^^^^^^^^^^^^^
+   |    Found:    () => Unit
+   |    Required: () ->? Unit
+   |
+   |    Note that the universal capability `cap`
+   |    cannot be included in capture set ?
+   |
+   | longer explanation available when compiling with `-explain`
 -- [E007] Type Mismatch Error: tests/neg-custom-args/captures/reaches.scala:64:27 --------------------------------------
 64 |    val f1: File^{id*} = id(f) // error, since now id(f): File^
    |                         ^^^^^

tests/neg-custom-args/captures/reaches.scala

@@ -54,7 +54,7 @@ class Id[-A,  +B >: A]():
 def test =
   val id: Id[Proc, Proc] = new Id[Proc, () -> Unit] // error
   usingFile: f =>
-    id(() => f.write())  // escape, if it was not for the error above
+    id(() => f.write())  // error
 
 def attack2 =
   val id: File^ -> File^ = x => x

tests/neg-custom-args/captures/unsound-reach-2.scala

@@ -20,7 +20,7 @@ def bad(): Unit =
 
   var escaped: File^{backdoor*} = null
   withFile("hello.txt"): f =>
-    boom.use(f): // error
+    boom.use(f):
       new Consumer[File^{backdoor*}]: // error
         def apply(f1: File^{backdoor*}) =
           escaped = f1

tests/neg-custom-args/captures/unsound-reach-3.scala

@@ -16,8 +16,8 @@ def bad(): Unit =
   val boom: Foo[File^{backdoor*}] = backdoor
 
   var escaped: File^{backdoor*} = null
-  withFile("hello.txt"): f =>
-    escaped = boom.use(f) // error
+  withFile("hello.txt"): f => // error
+    escaped = boom.use(f)
       // boom.use: (x: File^) -> File^{backdoor*}, it is a selection so reach capabilities are allowed
       // f: File^, so there is no reach capabilities
 

tests/neg-custom-args/captures/unsound-reach-4.check

@@ -1,5 +1,6 @@
--- Error: tests/neg-custom-args/captures/unsound-reach-4.scala:22:19 ---------------------------------------------------
-22 |    escaped = boom.use(f)  // error
-   |              ^^^^^^^^
-   |              Reach capability backdoor* and universal capability cap cannot both
-   |              appear in the type (x: F): box File^{backdoor*} of this expression
+-- Error: tests/neg-custom-args/captures/unsound-reach-4.scala:21:25 ---------------------------------------------------
+21 |  withFile("hello.txt"): f =>  // error
+   |                         ^
+   |                         Reach capability backdoor* and universal capability cap cannot both
+   |                         appear in the type (f: File^) ->{backdoor*} Unit of this expression
+22 |    escaped = boom.use(f)

tests/neg-custom-args/captures/unsound-reach-4.scala

@@ -18,5 +18,5 @@ def bad(): Unit =
   val boom: Foo[File^{backdoor*}] = backdoor
 
   var escaped: File^{backdoor*} = null
-  withFile("hello.txt"): f =>
-    escaped = boom.use(f)  // error
+  withFile("hello.txt"): f =>  // error
+    escaped = boom.use(f)