Better prediction after with in types

Also: generalize notion to get next token while suppressing
indentation insertion

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -920,6 +920,16 @@ object Parsers {
       val next = in.lookahead.token
       next == LBRACKET || next == LPAREN
 
+    private def withEndMigrationWarning(): Boolean =
+      migrateTo3
+      && {
+        warning(
+          em"""In Scala 3, `with` at the end of a line will start definitions,
+              |so it cannot be used in front of a parent constructor anymore.
+              |Place the `with` at the beginning of the next line instead.""")
+        true
+      }
+
     /** Does a template start after `with`? This is the case if either
      *   - the next token is `{`
      *   - the `with` is at the end of a line
@@ -931,17 +941,7 @@ object Parsers {
       val lookahead = in.LookaheadScanner()
       lookahead.nextToken()
       lookahead.token == LBRACE
-      || lookahead.isAfterLineEnd
-          && {
-            if migrateTo3 then
-              warning(
-                em"""In Scala 3, `with` at the end of a line will start definitions,
-                    |so it cannot be used in front of a parent constructor anymore.
-                    |Place the `with` at the beginning of the next line instead.""")
-              false
-            else
-              true
-          }
+      || lookahead.isAfterLineEnd && !withEndMigrationWarning()
       || (lookahead.isIdent || lookahead.token == THIS)
           && {
             lookahead.nextToken()
@@ -953,6 +953,20 @@ object Parsers {
             || lookahead.token == ARROW
           }
 
+    /** Does a refinement start after `with`? This is the case if either
+     *   - the next token is `{`
+     *   - the `with` is at the end of a line and is followed by a token that starts a declaration
+     */
+    def followingIsRefinementStart() =
+      val lookahead = in.LookaheadScanner()
+      lookahead.nextToken()
+      lookahead.token == LBRACE
+      || lookahead.isAfterLineEnd
+         && {
+           if lookahead.token == INDENT then lookahead.nextToken()
+           dclIntroTokens.contains(lookahead.token)
+         }
+
 /* --------- OPERAND/OPERATOR STACK --------------------------------------- */
 
     var opStack: List[OpInfo] = Nil
@@ -1584,11 +1598,8 @@ object Parsers {
     def withType(): Tree = withTypeRest(annotType())
 
     def withTypeRest(t: Tree): Tree =
-      def isRefinementStart =
-        val la = in.lookahead
-        la.isAfterLineEnd || la.token == LBRACE
-      if in.token == WITH && !isRefinementStart then
-        in.nextToken()
+      if in.token == WITH && !followingIsRefinementStart() then
+        in.nextTokenNoIndent()
         if sourceVersion.isAtLeast(`3.1`) then
           deprecationWarning(DeprecatedWithOperator())
         atSpan(startOffset(t)) { makeAndType(t, withType()) }
@@ -3858,8 +3869,7 @@ object Parsers {
               if (name != nme.ERROR)
                 self = makeSelfDef(name, tpt).withSpan(first.span)
           }
-          in.token = SELFARROW // suppresses INDENT insertion after `=>`
-          in.nextToken()
+          in.nextTokenNoIndent()
         }
         else {
           stats += first

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -323,6 +323,11 @@ object Scanners {
       printState()
     }
 
+    /** Like nextToken, but don't insert indent characters afterwards */
+    def nextTokenNoIndent(): Unit =
+      token = EMPTY // this will suppress newline and indent insertion
+      nextToken()
+
     final def printState() =
       if debugTokenStream && (showLookAheadOnDebug || !isInstanceOf[LookaheadScanner]) then
         print(s"[$show${if isInstanceOf[LookaheadScanner] then "(LA)" else ""}]")
@@ -493,8 +498,6 @@ object Scanners {
           if canStartIndentTokens.contains(lastToken) then
             currentRegion = Indented(nextWidth, Set(), lastToken, currentRegion)
             insert(INDENT, offset)
-          else if lastToken == SELFARROW then
-            currentRegion.knownWidth = nextWidth
         else if (lastWidth != nextWidth)
           errorButContinue(spaceTabMismatchMsg(lastWidth, nextWidth))
       currentRegion match {

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -204,7 +204,6 @@ object Tokens extends TokensCommon {
   final val QUOTE = 87;            enter(QUOTE, "'")
 
   final val COLONEOL = 88;         enter(COLONEOL, ":", ": at eol")
-  final val SELFARROW = 89;        enter(SELFARROW, "=>") // reclassified ARROW following self-type
 
   /** XML mode */
   final val XMLSTART = 99;         enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate

tests/neg/with-template.scala

@@ -11,4 +11,10 @@ object Test2 extends A with
     println("foo")
   }
 
+def foo: A with
+  B with
+  C = ???
+
+
+