Fix hole when resetting a typerstate

Besides the constraint, we also need to reset any instantiated type variables.
This is necessary for correctness and also to avoid uncollected garbage in constraints
when an entry is never dropped since the associated type variables are no longer
owned by the constraint.

Author: odersky

compiler/src/dotty/tools/dotc/core/TyperState.scala

@@ -22,6 +22,17 @@ object TyperState {
       .init(null, OrderingConstraint.empty)
       .setReporter(new ConsoleReporter())
       .setCommittable(true)
+
+  opaque type Snapshot = (Constraint, TypeVars)
+
+  extension (ts: TyperState)
+    def snapshot(): Snapshot = (ts.constraint, ts.ownedVars)
+
+    def resetTo(state: Snapshot)(using Context): Unit =
+      val (c, tvs) = state
+      for tv <- tvs do if tv.isInstantiated then tv.resetInst(ts)
+      ts.ownedVars = tvs
+      ts.constraint = c
 }
 
 class TyperState() {

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4404,6 +4404,10 @@ object Types {
           owningState1.ownedVars -= this
           owningState = null // no longer needed; null out to avoid a memory leak
 
+    private[core] def resetInst(ts: TyperState): Unit =
+      myInst = NoType
+      owningState = new WeakReference(ts)
+
     /** The state owning the variable. This is at first `creatorState`, but it can
      *  be changed to an enclosing state on a commit.
      */

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -3247,7 +3247,7 @@ class Typer extends Namer
         replaceSingletons(tp)
       }
       wtp.paramInfos.foreach(instantiate)
-      val constr = ctx.typerState.constraint
+      val saved = ctx.typerState.snapshot()
 
       def dummyArg(tp: Type) = untpd.Ident(nme.???).withTypeUnchecked(tp)
 
@@ -3347,8 +3347,9 @@ class Typer extends Namer
         if (propFail.exists) {
           // If there are several arguments, some arguments might already
           // have influenced the context, binding variables, but later ones
-          // might fail. In that case the constraint needs to be reset.
-          ctx.typerState.constraint = constr
+          // might fail. In that case the constraint and instantiated variables
+          // need to be reset.
+          ctx.typerState.resetTo(saved)
 
           // If method has default params, fall back to regular application
           // where all inferred implicits are passed as named args.

tests/neg/i9568.check

@@ -7,6 +7,6 @@
    |          .
    |          I found:
    |
-   |              Test.blaMonad[Nothing, S](Test.blaMonad[F, S])
+   |              Test.blaMonad[F, S](Test.blaMonad[F, S])
    |
-   |          But method blaMonad in object Test does not match type => Monad[Nothing].
+   |          But method blaMonad in object Test does not match type => Monad[F].