Eliminate WITHEOL token

Change parsing algorithm so that we do not need a separate WITHEOL token.
The grammar remains unaffected.

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -1283,7 +1283,7 @@ object Parsers {
 
     def possibleTemplateStart(isNew: Boolean = false): Unit =
       in.observeColonEOL()
-      if in.token == COLONEOL || in.token == WITHEOL then
+      if in.token == COLONEOL || in.token == WITH then
         if in.lookahead.isIdent(nme.end) then in.token = NEWLINE
         else
           in.nextToken()
@@ -3545,7 +3545,7 @@ object Parsers {
             ValDef(name, parents.head, subExpr())
           else
             DefDef(name, tparams, vparamss, parents.head, subExpr())
-        else if in.token != WITH && in.token != WITHEOL && parentsIsType then
+        else if in.token != WITH && parentsIsType then
           if name.isEmpty then
             syntaxError(em"anonymous given cannot be abstract")
           DefDef(name, tparams, vparamss, parents.head, EmptyTree)
@@ -3633,12 +3633,12 @@ object Parsers {
     /** `{`with` ConstrApp} but no EOL allowed after `with`.
      */
     def withConstrApps(): List[Tree] =
-      if in.token == WITH then
-        in.observeWithEOL() // converts token to WITHEOL if at end of line
-        if in.token == WITH && in.lookahead.token != LBRACE then
-          in.nextToken()
-          constrApp() :: withConstrApps()
-        else Nil
+      def isTemplateStart =
+        val la = in.lookahead
+        la.isAfterLineEnd || la.token == LBRACE
+      if in.token == WITH && !isTemplateStart then
+        in.nextToken()
+        constrApp() :: withConstrApps()
       else Nil
 
     /** Template          ::=  InheritClauses [TemplateBody]
@@ -3708,8 +3708,8 @@ object Parsers {
 
     /** with Template, with EOL <indent> interpreted */
     def withTemplate(constr: DefDef, parents: List[Tree]): Template =
-      if in.token != WITHEOL then accept(WITH)
-      possibleTemplateStart() // consumes a WITHEOL token
+      if in.token != WITH then syntaxError(em"`with` expected")
+      possibleTemplateStart() // consumes a WITH token
       val (self, stats) = templateBody()
       Template(constr, parents, Nil, self, stats)
         .withSpan(Span(constr.span.orElse(parents.head.span).start, in.lastOffset))

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -74,6 +74,9 @@ object Scanners {
 
     def isNestedStart = token == LBRACE || token == INDENT
     def isNestedEnd = token == RBRACE || token == OUTDENT
+
+    /** Is current token first one after a newline? */
+    def isAfterLineEnd: Boolean = lineOffset >= 0
   }
 
   abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData {
@@ -512,15 +515,12 @@ object Scanners {
          |Previous indent : $lastWidth
          |Latest indent   : $nextWidth"""
 
-    private def switchAtEOL(testToken: Token, eolToken: Token): Unit =
-      if token == testToken then
+    def observeColonEOL(): Unit =
+      if token == COLON then
         lookAhead()
         val atEOL = isAfterLineEnd || token == EOF
         reset()
-        if atEOL then token = eolToken
-
-    def observeColonEOL(): Unit = switchAtEOL(COLON, COLONEOL)
-    def observeWithEOL(): Unit = switchAtEOL(WITH, WITHEOL)
+        if atEOL then token = COLONEOL
 
     def observeIndented(): Unit =
       if indentSyntax && isNewLine then
@@ -610,9 +610,6 @@ object Scanners {
       }
     }
 
-    /** Is current token first one after a newline? */
-    def isAfterLineEnd: Boolean = lineOffset >= 0
-
     /** Is there a blank line between the current token and the last one?
      *  A blank line consists only of characters <= ' '.
      *  @pre  afterLineEnd().

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -204,8 +204,7 @@ object Tokens extends TokensCommon {
   final val QUOTE = 87;            enter(QUOTE, "'")
 
   final val COLONEOL = 88;         enter(COLONEOL, ":", ": at eol")
-  final val WITHEOL = 89;          enter(WITHEOL, "with", "with at eol")
-  final val SELFARROW = 90;        enter(SELFARROW, "=>") // reclassified ARROW following self-type
+  final val SELFARROW = 89;        enter(SELFARROW, "=>") // reclassified ARROW following self-type
 
   /** XML mode */
   final val XMLSTART = 99;         enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
@@ -277,8 +276,8 @@ object Tokens extends TokensCommon {
   final val closingRegionTokens = BitSet(RBRACE, RPAREN, RBRACKET, CASE) | statCtdTokens
 
   final val canStartIndentTokens: BitSet =
-    statCtdTokens | BitSet(COLONEOL, WITHEOL, EQUALS, ARROW, LARROW, WHILE, TRY, FOR, IF)
-      // `if` is excluded because it often comes after `else` which makes for awkward indentation rules  TODO: try to do without the exception
+    statCtdTokens | BitSet(COLONEOL, WITH, EQUALS, ARROW, LARROW, WHILE, TRY, FOR, IF)
+    // TODO: add THROW, CTXARROW
 
   /** Faced with the choice between a type and a formal parameter, the following
    *  tokens determine it's a formal parameter.