Stop at comma only if one could be expected

We now have the following handling of stop symbols in skip:

 - `)`, `]`, `}`: stop if there is a corresponding opening parenthesis
 - `,`          : stop if we are in a `commaSeparated` production
 - undent       : stop if indent width matches a previously seen indent width
 - `;`, nl      : stop unconditionally, the parser will sync after the current statement

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -225,7 +225,7 @@ object Parsers {
       || allowedMods.contains(in.token)
       || in.isSoftModifierInModifierPosition && !excludedSoftModifiers.contains(in.name)
 
-    def isStatSep: Boolean = in.isNewLine || in.token == SEMI
+    def isStatSep: Boolean = in.isStatSep
 
     /** A '$' identifier is treated as a splice if followed by a `{`.
      *  A longer identifier starting with `$` is treated as a splice/id combination
@@ -292,13 +292,13 @@ object Parsers {
       *
       * @return The offset at the start of the token to accept
       */
-    def accept(token: Int): Int = {
+    def accept(token: Int): Int =
       val offset = in.offset
-      if (in.token != token)
+      def assumedOutdent = token == OUTDENT && in.token == EOF
+      if in.token != token /*&& !assumedOutdent*/ then
         syntaxErrorOrIncomplete(ExpectedTokenButFound(token, in.token))
-      if (in.token == token) in.nextToken()
+      if in.token == token then in.nextToken()
       offset
-    }
 
     def accept(name: Name): Int = {
       val offset = in.offset
@@ -562,11 +562,13 @@ object Parsers {
       inBracesOrIndented(body, rewriteWithColon)
 
     def commaSeparated[T](part: () => T): List[T] =
-      val ts = new ListBuffer[T] += part()
-      while in.token == COMMA do
-        in.nextToken()
-        ts += part()
-      ts.toList
+      in.currentRegion.withCommasExpected {
+        val ts = new ListBuffer[T] += part()
+        while in.token == COMMA do
+          in.nextToken()
+          ts += part()
+        ts.toList
+      }
 
     def inSepRegion[T](f: Region => Region)(op: => T): T =
       val cur = in.currentRegion

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -67,6 +67,7 @@ object Scanners {
     }
 
     def isNewLine = token == NEWLINE || token == NEWLINES
+    def isStatSep = isNewLine || token == SEMI
     def isIdent = token == IDENTIFIER || token == BACKQUOTED_IDENT
     def isIdent(name: Name) = token == IDENTIFIER && this.name == name
 
@@ -291,8 +292,9 @@ object Scanners {
       def atStop =
         token == EOF
         || (currentRegion eq lastRegion)
-            && (stopSkipTokens.contains(token)
+            && (isStatSep
                 || closingParens.contains(token) && lastRegion.toList.exists(_.closedBy == token)
+                || token == COMMA && lastRegion.toList.exists(_.commasExpected)
                 || token == OUTDENT && indentWidth(offset) < lastKnownIndentWidth)
           // stop at OUTDENT if the new indentwidth is smaller than the indent width of
           // currentRegion. This corrects for the problem that sometimes we don't see an INDENT
@@ -1570,6 +1572,17 @@ object Scanners {
     protected def coversIndent(w: IndentWidth): Boolean =
       knownWidth != null && w == indentWidth
 
+    private var myCommasExpected: Boolean = false
+
+    inline def withCommasExpected[T](inline op: => T): T =
+      val saved = myCommasExpected
+      myCommasExpected = true
+      val res = op
+      myCommasExpected = false
+      res
+
+    def commasExpected = myCommasExpected
+
     def toList: List[Region] =
       this :: (if outer == null then Nil else outer.toList)
 

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -288,7 +288,6 @@ object Tokens extends TokensCommon {
   final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, THROW, GIVEN, VAL, THIS)
 
   final val closingParens = BitSet(RPAREN, RBRACKET, RBRACE)
-  final val stopSkipTokens = BitSet(SEMI, COMMA, NEWLINE, NEWLINES)
 
   final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent, nme.infix)
 }

tests/neg/i1679.scala

@@ -1,5 +1,5 @@
 class A[T]
 object o {
   // Testing compiler crash, this test should be modified when named type argument are completely implemented
-  val x: A[T=Int, T=Int] = ??? // error: ']' expected, but '=' found // error
+  val x: A[T=Int, T=Int] = ??? // error: ']' expected, but '=' found
 }