Tighten the screws a bit more to seal the soundness hole for reach capabilities (#20056)

Author: odersky

compiler/src/dotty/tools/dotc/cc/CaptureOps.scala

@@ -289,7 +289,7 @@ extension (tp: Type)
       var ok = true
       def traverse(t: Type): Unit =
         if ok then
-          t match
+          t.dealias match
             case CapturingType(_, cs) if cs.isUniversal && variance <= 0 =>
               ok = false
             case _ =>

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -259,7 +259,7 @@ class CheckCaptures extends Recheck, SymTransformer:
             var refVariances: Map[Boolean, Int] = Map.empty
             var seenReach: CaptureRef | Null = null
             def traverse(tp: Type) =
-                tp match
+                tp.dealias match
                 case CapturingType(parent, refs) =>
                     traverse(parent)
                     for ref <- refs.elems do

tests/neg/unsound-reach-4.check

@@ -0,0 +1,5 @@
+-- Error: tests/neg/unsound-reach-4.scala:20:19 ------------------------------------------------------------------------
+20 |    escaped = boom.use(f)  // error
+   |              ^^^^^^^^
+   |              Reach capability backdoor* and universal capability cap cannot both
+   |              appear in the type (x: F): box File^{backdoor*} of this expression

tests/neg/unsound-reach-4.scala

@@ -0,0 +1,20 @@
+import language.experimental.captureChecking
+trait File:
+  def close(): Unit
+
+def withFile[R](path: String)(op: File^ => R): R = ???
+
+type F = File^
+
+trait Foo[+X]:
+  def use(x: F): X
+class Bar extends Foo[File^]:
+  def use(x: F): File^ = x
+
+def bad(): Unit =
+  val backdoor: Foo[File^] = new Bar
+  val boom: Foo[File^{backdoor*}] = backdoor
+
+  var escaped: File^{backdoor*} = null
+  withFile("hello.txt"): f =>
+    escaped = boom.use(f)  // error