Make treatment of do language version dependent

`do` can start an expression or a statement only under -language:Scala2.
Maintaining this distinction is important in order not to insert spurious
newlines in front of `do` in a `while` or `for`.

Author: odersky

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -207,7 +207,7 @@ object Parsers {
     } && !in.isSoftModifierInModifierPosition
 
     def isExprIntro: Boolean =
-      canStartExpressionTokens.contains(in.token) && !in.isSoftModifierInModifierPosition
+      in.canStartExprTokens.contains(in.token) && !in.isSoftModifierInModifierPosition
 
     def isDefIntro(allowedMods: BitSet, excludedSoftModifiers: Set[TermName] = Set.empty): Boolean =
       in.token == AT ||
@@ -1231,6 +1231,10 @@ object Parsers {
       // note: next is defined here because current == NEWLINE
       if (in.token == NEWLINE && in.next.token == token) in.nextToken()
 
+    def newLinesOptWhenFollowedBy(name: Name): Unit =
+      if in.isNewLine && in.next.token == IDENTIFIER && in.next.name == name then
+        in.nextToken()
+
     def newLineOptWhenFollowing(p: Int => Boolean): Unit =
       // note: next is defined here because current == NEWLINE
       if (in.token == NEWLINE && p(in.next.token)) newLineOpt()
@@ -1246,7 +1250,7 @@ object Parsers {
     }
 
     def possibleTemplateStart(): Unit = {
-      in.observeIndented(unless = noIndentTemplateTokens, unlessSoftKW = nme.derives)
+      in.observeIndented()
       newLineOptWhenFollowedBy(LBRACE)
     }
 
@@ -1651,7 +1655,7 @@ object Parsers {
     def toBeContinued(altToken: Token): Boolean =
       if in.token == altToken || in.isNewLine || in.isScala2Mode then
         false // a newline token means the expression is finished
-      else if !canStartStatTokens.contains(in.token)
+      else if !in.canStartStatTokens.contains(in.token)
               || in.isLeadingInfixOperator(inConditional = true)
       then
         true
@@ -1852,20 +1856,20 @@ object Parsers {
           }
         }
       case _ =>
-        if (isIdent(nme.inline) && !in.inModifierPosition() && in.lookaheadIn(canStartExpressionTokens)) {
+        if isIdent(nme.inline)
+           && !in.inModifierPosition()
+           && in.lookaheadIn(in.canStartExprTokens)
+        then
           val start = in.skipToken()
-          in.token match {
+          in.token match
             case IF =>
               ifExpr(start, InlineIf)
             case _ =>
               val t = postfixExpr()
               if (in.token == MATCH) matchExpr(t, start, InlineMatch)
-              else {
+              else
                 syntaxErrorOrIncomplete(i"`match` or `if` expected but ${in.token} found")
                 t
-              }
-          }
-        }
         else expr1Rest(postfixExpr(), location)
     }
 
@@ -2012,7 +2016,7 @@ object Parsers {
     def postfixExpr(): Tree = postfixExprRest(prefixExpr())
 
     def postfixExprRest(t: Tree): Tree =
-      infixOps(t, canStartExpressionTokens, prefixExpr, maybePostfix = true)
+      infixOps(t, in.canStartExprTokens, prefixExpr, maybePostfix = true)
 
     /** PrefixExpr   ::= [`-' | `+' | `~' | `!'] SimpleExpr
     */
@@ -2209,7 +2213,7 @@ object Parsers {
             lookahead.nextToken()
             lookahead.token != COLON
           }
-          else canStartExpressionTokens.contains(lookahead.token)
+          else in.canStartExprTokens.contains(lookahead.token)
         }
       }
       if (in.token == LPAREN && (!inClassConstrAnnots || isLegalAnnotArg))
@@ -2338,7 +2342,7 @@ object Parsers {
                 dropParensOrBraces(start, if (in.token == YIELD || in.token == DO) "" else "do")
               }
             }
-            in.observeIndented(unless = noIndentAfterEnumeratorTokens)
+            in.observeIndented()
             res
           }
           else {
@@ -2484,7 +2488,7 @@ object Parsers {
     /**  InfixPattern ::= SimplePattern {id [nl] SimplePattern}
      */
     def infixPattern(): Tree =
-      infixOps(simplePattern(), canStartExpressionTokens, simplePattern, isOperator = in.name != nme.raw.BAR)
+      infixOps(simplePattern(), in.canStartExprTokens, simplePattern, isOperator = in.name != nme.raw.BAR)
 
     /** SimplePattern    ::= PatVar
      *                    |  Literal
@@ -3470,6 +3474,7 @@ object Parsers {
     /** Template          ::=  InheritClauses [TemplateBody]
      */
     def template(constr: DefDef, isEnum: Boolean = false): Template = {
+      newLinesOptWhenFollowedBy(nme.derives)
       val (parents, derived) = inheritClauses()
       possibleTemplateStart()
       if (isEnum) {
@@ -3482,10 +3487,11 @@ object Parsers {
     /** TemplateOpt = [Template]
      */
     def templateOpt(constr: DefDef): Template =
-      possibleTemplateStart()
+      newLinesOptWhenFollowedBy(nme.derives)
       if in.token == EXTENDS || isIdent(nme.derives) then
         template(constr)
       else
+        possibleTemplateStart()
         if in.isNestedStart then
           template(constr)
         else

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -385,7 +385,7 @@ object Scanners {
         lookahead.allowLeadingInfixOperators = false
           // force a NEWLINE a after current token if it is on its own line
         lookahead.nextToken()
-        canStartExpressionTokens.contains(lookahead.token)
+        canStartExprTokens.contains(lookahead.token)
       }
       && {
         if isScala2Mode || oldSyntax && !rewrite then
@@ -495,10 +495,11 @@ object Scanners {
           indentPrefix = LBRACE
         case _ =>
       }
-      if (newlineIsSeparating &&
-          canEndStatTokens.contains(lastToken) &&
-          canStartStatTokens.contains(token) &&
-          !isLeadingInfixOperator())
+      if newlineIsSeparating
+         && canEndStatTokens.contains(lastToken)
+         && canStartStatTokens.contains(token)
+         && !isLeadingInfixOperator()
+      then
         insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset)
       else if indentIsSignificant then
         if nextWidth < lastWidth
@@ -541,29 +542,19 @@ object Scanners {
          |Previous indent : $lastWidth
          |Latest indent   : $nextWidth"""
 
-    def observeIndented(
-      unless: BitSet = BitSet.empty,
-      unlessSoftKW: TermName = EmptyTermName): Unit
-    =
-      if (indentSyntax && isAfterLineEnd && token != INDENT) {
-        val nextOffset = if (isNewLine) next.offset else offset
-        val nextToken = if (isNewLine) next.token else token
-        val nextWidth = indentWidth(nextOffset)
-        val lastWidth = currentRegion match {
+    def observeIndented(): Unit =
+      if indentSyntax && isNewLine then
+        val nextWidth = indentWidth(next.offset)
+        val lastWidth = currentRegion match
           case r: Indented => r.width
           case r: InBraces => r.width
           case _ => nextWidth
-        }
 
-        if (lastWidth < nextWidth
-           && !unless.contains(nextToken)
-           && (unlessSoftKW.isEmpty || token != IDENTIFIER || name != unlessSoftKW)) {
+        if lastWidth < nextWidth then
           currentRegion = Indented(nextWidth, Set(), COLONEOL, currentRegion)
-          if (!isNewLine) next.copyFrom(this)
-          offset = nextOffset
+          offset = next.offset
           token = INDENT
-        }
-      }
+    end observeIndented
 
     /** - Join CASE + CLASS => CASECLASS, CASE + OBJECT => CASEOBJECT, SEMI + ELSE => ELSE, COLON + <EOL> => COLONEOL
      *  - Insert missing OUTDENTs at EOF
@@ -1011,6 +1002,12 @@ object Scanners {
     def isNestedStart = token == LBRACE || token == INDENT
     def isNestedEnd = token == RBRACE || token == OUTDENT
 
+    def canStartStatTokens =
+      if isScala2Mode then canStartStatTokens2 else canStartStatTokens3
+
+    def canStartExprTokens =
+      if isScala2Mode then canStartExprTokens2 else canStartExprTokens3
+
 // Literals -----------------------------------------------------------------
 
     private def getStringLit() = {

compiler/src/dotty/tools/dotc/parsing/Tokens.scala

@@ -219,8 +219,10 @@ object Tokens extends TokensCommon {
   final val atomicExprTokens: TokenSet = literalTokens | identifierTokens | BitSet(
     USCORE, NULL, THIS, SUPER, TRUE, FALSE, RETURN, QUOTEID, XMLSTART)
 
-  final val canStartExpressionTokens: TokenSet = atomicExprTokens | BitSet(
-    LBRACE, LPAREN, INDENT, QUOTE, IF, DO, WHILE, FOR, NEW, TRY, THROW)
+  final val canStartExprTokens3: TokenSet = atomicExprTokens | BitSet(
+    LBRACE, LPAREN, INDENT, QUOTE, IF, WHILE, FOR, NEW, TRY, THROW)
+
+  final val canStartExprTokens2: TokenSet = canStartExprTokens3 | BitSet(DO)
 
   final val canStartTypeTokens: TokenSet = literalTokens | identifierTokens | BitSet(
     THIS, SUPER, USCORE, LPAREN, AT)
@@ -247,7 +249,9 @@ object Tokens extends TokensCommon {
   /** Is token only legal as start of statement (eof also included)? */
   final val mustStartStatTokens: TokenSet = defIntroTokens | modifierTokens | BitSet(IMPORT, EXPORT, PACKAGE)
 
-  final val canStartStatTokens: TokenSet = canStartExpressionTokens | mustStartStatTokens | BitSet(
+  final val canStartStatTokens2: TokenSet = canStartExprTokens2 | mustStartStatTokens | BitSet(
+    AT, CASE)
+  final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet(
     AT, CASE)
 
   final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet(
@@ -280,10 +284,6 @@ object Tokens extends TokensCommon {
    */
   final val startParamOrGivenTypeTokens: BitSet = startParamTokens | BitSet(GIVEN, ERASED)
 
-  final val noIndentTemplateTokens = BitSet(EXTENDS)
-  final val noIndentAfterConditionTokens = BitSet(THEN, DO)
-  final val noIndentAfterEnumeratorTokens = BitSet(YIELD, DO)
-
   final val scala3keywords = BitSet(ENUM, ERASED, GIVEN)
 
   final val softModifierNames = Set(nme.inline, nme.opaque)

compiler/src/dotty/tools/dotc/util/kwords.sc

@@ -13,6 +13,6 @@ object kwords {
                                                   //| atch, lazy, then, forSome, _, :, =, <-, =>, ';', ';', <:, >:, #, @, <%)
   keywords.toList.filter(kw => tokenString(kw) == null)
                                                   //> res1: List[Int] = List()
-    canStartStatTokens contains CASE              //> res2: Boolean = false
-                                                  
+  canStartStatTokens3 contains CASE               //> res2: Boolean = false
+
 }

tests/pos/indent4.scala

@@ -22,3 +22,14 @@ object testindent
     val y = x
     println(y)
 
+  while true
+  do println(1)
+
+  for i <- List(1, 2, 3)
+  do println(i)
+
+  while (true)
+  do println(1)
+
+  for (i <- List(1, 2, 3))
+  do println(i)