Treat end markers as separate tokens

Detect end markers in Scanner

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -167,6 +167,7 @@ object Parsers {
   class Parser(source: SourceFile)(using Context) extends ParserCommon(source) {
 
     val in: Scanner = new Scanner(source)
+    // in.debugTokenStream = true    // uncomment to see the token stream of the standard scanner, but not syntax highlighting
 
     /** This is the general parse entry point.
      *  Overridden by ScriptParser
@@ -1242,7 +1243,7 @@ object Parsers {
     def possibleTemplateStart(isNew: Boolean = false): Unit =
       in.observeColonEOL()
       if in.token == COLONEOL then
-        if in.lookahead.isIdent(nme.end) then in.token = NEWLINE
+        if in.lookahead.token == END then in.token = NEWLINE
         else
           in.nextToken()
           if in.token != INDENT && in.token != LBRACE then
@@ -1272,25 +1273,12 @@ object Parsers {
         case _: (ForYield | ForDo) => in.token == FOR
         case _ => false
 
-      if isIdent(nme.end) then
-        val start = in.offset
-        val isEndMarker =
-          val endLine = source.offsetToLine(start)
-          val lookahead = in.LookaheadScanner()
-          lookahead.nextToken()
-          source.offsetToLine(lookahead.offset) == endLine
-          && endMarkerTokens.contains(in.token)
-          && {
-            lookahead.nextToken()
-            lookahead.token == EOF
-            || source.offsetToLine(lookahead.offset) > endLine
-          }
-        if isEndMarker then
-          in.nextToken()
-          if stats.isEmpty || !matches(stats.last) then
-            syntaxError("misaligned end marker", Span(start, in.lastCharOffset))
-          in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion
-          in.nextToken()
+      if in.token == END then
+        val start = in.skipToken()
+        if stats.isEmpty || !matches(stats.last) then
+          syntaxError("misaligned end marker", Span(start, in.lastCharOffset))
+        in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion
+        in.nextToken()
     end checkEndMarker
 
 /* ------------- TYPES ------------------------------------------------------ */

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -135,14 +135,16 @@ object Scanners {
       */
     protected def putChar(c: Char): Unit = litBuf.append(c)
 
-    /** Clear buffer and set name and token */
-    def finishNamed(idtoken: Token = IDENTIFIER, target: TokenData = this): Unit = {
+    /** Clear buffer and set name and token
+     *  If `target` is different from `this`, don't treat identifiers as end tokens
+     */
+    def finishNamed(idtoken: Token = IDENTIFIER, target: TokenData = this): Unit =
       target.name = termName(litBuf.chars, 0, litBuf.length)
       litBuf.clear()
       target.token = idtoken
-      if (idtoken == IDENTIFIER)
-        target.token = toToken(target.name)
-    }
+      if idtoken == IDENTIFIER then
+        val converted = toToken(target.name)
+        if converted != END || (target eq this) then target.token = converted
 
     /** The token for given `name`. Either IDENTIFIER or a keyword. */
     def toToken(name: SimpleName): Token
@@ -656,6 +658,8 @@ object Scanners {
                 () /* skip the trailing comma */
               else
                 reset()
+        case END =>
+          if !isEndMarker then token = IDENTIFIER
         case COLON =>
           if fewerBracesEnabled then observeColonEOL()
         case RBRACE | RPAREN | RBRACKET =>
@@ -666,6 +670,21 @@ object Scanners {
       }
     }
 
+    protected def isEndMarker: Boolean =
+      if isAfterLineEnd then
+        val endLine = source.offsetToLine(offset)
+        val lookahead = new LookaheadScanner():
+          override def isEndMarker = false
+        lookahead.nextToken()
+        if endMarkerTokens.contains(lookahead.token)
+          && source.offsetToLine(lookahead.offset) == endLine
+        then
+          lookahead.nextToken()
+          if lookahead.token == EOF
+          || source.offsetToLine(lookahead.offset) > endLine
+          then return true
+      false
+
     /** Is there a blank line between the current token and the last one?
      *  A blank line consists only of characters <= ' '.
      *  @pre  afterLineEnd().

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -183,6 +183,7 @@ object Tokens extends TokensCommon {
   final val GIVEN = 63;            enter(GIVEN, "given")
   final val EXPORT = 64;           enter(EXPORT, "export")
   final val MACRO = 65;            enter(MACRO, "macro") // TODO: remove
+  final val END = 66;              enter(END, "end")
 
   /** special symbols */
   final val NEWLINE = 78;          enter(NEWLINE, "end of statement", "new line")
@@ -207,7 +208,7 @@ object Tokens extends TokensCommon {
   /** XML mode */
   final val XMLSTART = 99;         enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
 
-  final val alphaKeywords: TokenSet = tokenRange(IF, MACRO)
+  final val alphaKeywords: TokenSet = tokenRange(IF, END)
   final val symbolicKeywords: TokenSet = tokenRange(USCORE, CTXARROW)
   final val keywords: TokenSet = alphaKeywords | symbolicKeywords
 
@@ -256,7 +257,7 @@ object Tokens extends TokensCommon {
   final val canStartStatTokens2: TokenSet = canStartExprTokens2 | mustStartStatTokens | BitSet(
     AT, CASE)
   final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet(
-    AT, CASE)
+    AT, CASE, END)
 
   final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet(TYPE, GIVEN, RPAREN, RBRACE, RBRACKET, OUTDENT)
 

tests/neg/i12150.check

@@ -0,0 +1,6 @@
+-- [E018] Syntax Error: tests/neg/i12150.scala:1:13 --------------------------------------------------------------------
+1 |def f: Unit = // error
+  |             ^
+  |             expression expected but end found
+
+longer explanation available when compiling with `-explain`

tests/neg/i12150.scala

@@ -0,0 +1,3 @@
+def f: Unit = // error
+
+end f