Skip to content

Commit 8942ce5

Browse files
oderskyodersky
committed
Improve skipping on syntax errors
Simplify and improve the algorithm used for skipping tokens when a syntax error is encountered. The new algorithm is based on the region stack maintained in Scanner instead of maintaining separate counters for open parentheses. This improves the precision of skipping in general, since there's is no chance that the two datastructures get out of sync. It seems overall we produce fewer follow on error after a syntax error that way.
1 parent 98e3839 commit 8942ce5

File tree

15 files changed

+57
-103
lines changed

15 files changed

+57
-103
lines changed

‎compiler/src/dotty/tools/dotc/parsing/Parsers.scala‎

Lines changed: 9 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,6 @@ object Parsers {
3737

3838
case class OpInfo(operand: Tree, operator: Ident, offset: Offset)
3939

40-
class ParensCounters {
41-
private var parCounts = new Array[Int](lastParen - firstParen)
42-
43-
def count(tok: Token): Int = parCounts(tok - firstParen)
44-
def change(tok: Token, delta: Int): Unit = parCounts(tok - firstParen) += delta
45-
def nonePositive: Boolean = parCounts forall (_ <= 0)
46-
}
47-
4840
enum Location(val inParens: Boolean, val inPattern: Boolean, val inArgs: Boolean):
4941
case InParens extends Location(true, false, false)
5042
case InArgs extends Location(true, false, true)
@@ -173,8 +165,6 @@ object Parsers {
173165

174166
val in: Scanner = new Scanner(source)
175167

176-
val openParens: ParensCounters = new ParensCounters
177-
178168
/** This is the general parse entry point.
179169
* Overridden by ScriptParser
180170
*/
@@ -261,55 +251,14 @@ object Parsers {
261251
}
262252

263253
/** Skip on error to next safe point.
264-
* Safe points are:
265-
* - Closing braces, provided they match an opening brace before the error point.
266-
* - Closing parens and brackets, provided they match an opening parent or bracket
267-
* before the error point and there are no intervening other kinds of parens.
268-
* - Semicolons and newlines, provided there are no intervening braces.
269-
* - Definite statement starts on new lines, provided they are not more indented
270-
* than the last known statement start before the error point.
271-
*/
272-
protected def skip(): Unit = {
273-
val skippedParens = new ParensCounters
274-
while (true) {
275-
(in.token: @switch) match {
276-
case EOF =>
277-
return
278-
case SEMI | NEWLINE | NEWLINES =>
279-
if (skippedParens.count(LBRACE) == 0) return
280-
case RBRACE =>
281-
if (openParens.count(LBRACE) > 0 && skippedParens.count(LBRACE) == 0)
282-
return
283-
skippedParens.change(LBRACE, -1)
284-
case RPAREN =>
285-
if (openParens.count(LPAREN) > 0 && skippedParens.nonePositive)
286-
return
287-
skippedParens.change(LPAREN, -1)
288-
case RBRACKET =>
289-
if (openParens.count(LBRACKET) > 0 && skippedParens.nonePositive)
290-
return
291-
skippedParens.change(LBRACKET, -1)
292-
case OUTDENT =>
293-
if (openParens.count(INDENT) > 0 && skippedParens.count(INDENT) == 0)
294-
return
295-
skippedParens.change(INDENT, -1)
296-
case LBRACE =>
297-
skippedParens.change(LBRACE, +1)
298-
case LPAREN =>
299-
skippedParens.change(LPAREN, +1)
300-
case LBRACKET=>
301-
skippedParens.change(LBRACKET, +1)
302-
case INDENT =>
303-
skippedParens.change(INDENT, +1)
304-
case _ =>
305-
if (mustStartStat &&
306-
in.isAfterLineEnd &&
307-
isLeqIndented(in.offset, lastStatOffset max 0))
308-
return
309-
}
254+
*/
255+
protected def skip(): Unit =
256+
val lastRegion = in.currentRegion
257+
def atStop =
258+
in.token == EOF
259+
|| skipStopTokens.contains(in.token) && (in.currentRegion eq lastRegion)
260+
while !atStop do
310261
in.nextToken()
311-
}
312-
}
313262

314263
def warning(msg: Message, sourcePos: SourcePosition): Unit =
315264
report.warning(msg, sourcePos)
@@ -557,15 +506,9 @@ object Parsers {
557506

558507
/* -------- COMBINATORS -------------------------------------------------------- */
559508

560-
def enclosed[T](tok: Token, body: => T): T = {
509+
def enclosed[T](tok: Token, body: => T): T =
561510
accept(tok)
562-
openParens.change(tok, 1)
563-
try body
564-
finally {
565-
accept(tok + 1)
566-
openParens.change(tok, -1)
567-
}
568-
}
511+
try body finally accept(tok + 1)
569512

570513
def inParens[T](body: => T): T = enclosed(LPAREN, body)
571514
def inBraces[T](body: => T): T = enclosed(LBRACE, body)
@@ -1429,7 +1372,6 @@ object Parsers {
14291372
functionRest(Nil)
14301373
}
14311374
else {
1432-
openParens.change(LPAREN, 1)
14331375
imods = modifiers(funTypeArgMods)
14341376
val paramStart = in.offset
14351377
val ts = funArgType() match {
@@ -1441,7 +1383,6 @@ object Parsers {
14411383
case t =>
14421384
funArgTypesRest(t, funArgType)
14431385
}
1444-
openParens.change(LPAREN, -1)
14451386
accept(RPAREN)
14461387
if isValParamList || in.token == ARROW || in.token == CTXARROW then
14471388
functionRest(ts)
@@ -2149,14 +2090,12 @@ object Parsers {
21492090
if in.token == RPAREN then
21502091
Nil
21512092
else
2152-
openParens.change(LPAREN, 1)
21532093
var mods1 = mods
21542094
if in.token == ERASED then mods1 = addModifier(mods1)
21552095
try
21562096
commaSeparated(() => binding(mods1))
21572097
finally
21582098
accept(RPAREN)
2159-
openParens.change(LPAREN, -1)
21602099
else {
21612100
val start = in.offset
21622101
val name = bindingName()
@@ -2504,7 +2443,6 @@ object Parsers {
25042443
val enums =
25052444
if (leading == LBRACE || leading == LPAREN && followingIsEnclosedGenerators()) {
25062445
in.nextToken()
2507-
openParens.change(leading, 1)
25082446
val res =
25092447
if (leading == LBRACE || in.token == CASE)
25102448
enumerators()
@@ -2514,7 +2452,6 @@ object Parsers {
25142452
if (in.token == RPAREN || pats.length > 1) {
25152453
wrappedEnums = false
25162454
accept(RPAREN)
2517-
openParens.change(LPAREN, -1)
25182455
atSpan(start) { makeTupleOrParens(pats) } // note: alternatives `|' need to be weeded out by typer.
25192456
}
25202457
else pats.head
@@ -2523,7 +2460,6 @@ object Parsers {
25232460
if (wrappedEnums) {
25242461
val closingOnNewLine = in.isAfterLineEnd
25252462
accept(leading + 1)
2526-
openParens.change(leading, -1)
25272463
def hasMultiLineEnum =
25282464
res.exists { t =>
25292465
val pos = t.sourcePos

‎compiler/src/dotty/tools/dotc/parsing/Scanners.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1383,6 +1383,22 @@ object Scanners {
13831383
private def useOuterWidth(): Unit =
13841384
if enclosing.knownWidth == null then enclosing.useOuterWidth()
13851385
knownWidth = enclosing.knownWidth
1386+
1387+
private def delimiter = this match
1388+
case _: InString => "}(in string)"
1389+
case InParens(LPAREN, _) => ")"
1390+
case InParens(LBRACKET, _) => "]"
1391+
case _: InBraces => "}"
1392+
case _: InCase => "=>"
1393+
case _: Indented => "UNDENT"
1394+
1395+
/** Show open regions as list of lines with decreasing indentations */
1396+
def visualize: String =
1397+
indentWidth.toPrefix
1398+
+ delimiter
1399+
+ outer.match
1400+
case null => ""
1401+
case next: Region => "\n" + next.visualize
13861402
end Region
13871403

13881404
case class InString(multiLine: Boolean, outer: Region) extends Region

‎compiler/src/dotty/tools/dotc/parsing/Tokens.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,5 +287,7 @@ object Tokens extends TokensCommon {
287287

288288
final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, THROW, GIVEN, VAL, THIS)
289289

290+
final val skipStopTokens = BitSet(SEMI, NEWLINE, NEWLINES, RBRACE, RPAREN, RBRACKET, OUTDENT)
291+
290292
final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent, nme.infix)
291293
}

‎tests/neg/errpos.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
object Test {
22
val x = // error: expression expected
3-
val y = 2 // error: ';' expected
3+
val y = 2
44

55
val z = // error: expression expected
66

77
// ...
8-
val a = 3 // error: ';' expected
8+
val a = 3
99

1010
val b = type // error: expression expected (on "type")
1111

‎tests/neg/extensions-can-only-have-defs.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
object Test {
22
extension [T] (t: T) {
3-
val v: T = ??? // error // error
3+
val v: T = ??? // error
44
}
55
}

‎tests/neg/i10546.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ object test:
22
def times(num : Int)(block : => Unit) : Unit = ()
33
times(10): println("ah") // error: end of statement expected but '(' found // error
44

5-
def foo: Set(Int) = Set(1) // error: end of statement expected but '(' found // error // error
5+
def foo: Set(Int) = Set(1)

‎tests/neg/i4373.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class A4 extends _ with Base // error
1616
object Test {
1717
type T1 = _ // error
1818
type T2 = _[Int] // error // error
19-
type T3 = _ { type S } // error
19+
type T3 = _ { type S } // error // error
2020
type T4 = [X] =>> _ // error
2121

2222
// Open questions:

‎tests/neg/i4934.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ object Main {
88

99
object Foo {
1010
val a = ""); // error: `}` expected but `)` found
11-
} // error: eof expected
11+
}

‎tests/neg/i5004.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
object i0 {
22
1 match {
3-
def this(): Int // error // error
4-
def this() // error
5-
} // error
3+
def this(): Int // error
4+
def this()
5+
}
66
}

‎tests/neg/i5032b.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
class a@
2-
class b@ // error // error
2+
class b@ // error

0 commit comments

Comments
 (0)