Skip to content

Commit 7e8e5a7

Browse files
lihaoyilihaoyi
committed
.
1 parent b181814 commit 7e8e5a7

File tree

2 files changed

+195
-29
lines changed

2 files changed

+195
-29
lines changed

‎compiler/src/dotty/tools/dotc/parsing/Parsers.scala‎

Lines changed: 193 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,46 @@ object Parsers {
13491349
else
13501350
literal(inTypeOrSingleton = true)
13511351

1352+
/** Dedent a string literal by removing common leading whitespace.
1353+
* The amount of whitespace to remove is determined by the indentation
1354+
* of the last line (which should contain only whitespace before the
1355+
* closing delimiter).
1356+
*/
1357+
private def dedentString(str: String): String = {
1358+
if (str.isEmpty) return str
1359+
1360+
// Find the last line (should be just whitespace before closing delimiter)
1361+
val lastNewlineIdx = str.lastIndexOf('\n')
1362+
if (lastNewlineIdx < 0) {
1363+
// No newlines, return as-is (shouldn't happen for valid dedented strings)
1364+
return str
1365+
}
1366+
1367+
// Extract the indentation from the last line
1368+
val closingIndent = str.substring(lastNewlineIdx + 1)
1369+
1370+
// Split into lines
1371+
val lines = str.split("\n", -1) // -1 to keep trailing empty strings
1372+
1373+
// Process all lines except the last (which is just the closing indentation)
1374+
val dedented = lines.dropRight(1).map { line =>
1375+
if (line.startsWith(closingIndent)) {
1376+
line.substring(closingIndent.length)
1377+
} else if (line.trim.isEmpty) {
1378+
// Empty or whitespace-only lines
1379+
""
1380+
} else {
1381+
// Line doesn't start with the closing indentation, keep as-is
1382+
line
1383+
}
1384+
}
1385+
1386+
// Drop the first line if it's empty (the newline after opening delimiter)
1387+
val result = if (dedented.headOption.contains("")) dedented.drop(1) else dedented
1388+
1389+
result.mkString("\n")
1390+
}
1391+
13521392
/** Literal ::= SimpleLiteral
13531393
* | processedStringLiteral
13541394
* | symbolLiteral
@@ -1377,7 +1417,15 @@ object Parsers {
13771417
case FLOATLIT => floatFromDigits(digits)
13781418
case DOUBLELIT | DECILIT | EXPOLIT => doubleFromDigits(digits)
13791419
case CHARLIT => in.strVal.head
1380-
case STRINGLIT | STRINGPART => in.strVal
1420+
case STRINGLIT | STRINGPART =>
1421+
// Check if this is a dedented string (non-interpolated)
1422+
// For non-interpolated dedented strings, check if the token starts with '''
1423+
val str = in.strVal
1424+
if (token == STRINGLIT && !inStringInterpolation && isDedentedStringLiteral(negOffset)) {
1425+
dedentString(str)
1426+
} else {
1427+
str
1428+
}
13811429
case TRUE => true
13821430
case FALSE => false
13831431
case NULL => null
@@ -1391,6 +1439,15 @@ object Parsers {
13911439
Literal(Constant(value))
13921440
}
13931441

1442+
/** Check if a string literal at the given offset is a dedented string */
1443+
def isDedentedStringLiteral(offset: Int): Boolean = {
1444+
val buf = in.buf
1445+
offset + 2 < buf.length &&
1446+
buf(offset) == '\'' &&
1447+
buf(offset + 1) == '\'' &&
1448+
buf(offset + 2) == '\''
1449+
}
1450+
13941451
if (inStringInterpolation) {
13951452
val t = in.token match {
13961453
case STRINGLIT | STRINGPART =>
@@ -1447,40 +1504,147 @@ object Parsers {
14471504
in.charOffset + 1 < in.buf.length &&
14481505
in.buf(in.charOffset) == '"' &&
14491506
in.buf(in.charOffset + 1) == '"'
1507+
val isDedented =
1508+
in.charOffset + 2 < in.buf.length &&
1509+
in.buf(in.charOffset) == '\'' &&
1510+
in.buf(in.charOffset + 1) == '\'' &&
1511+
in.buf(in.charOffset + 2) == '\''
1512+
14501513
in.nextToken()
1451-
def nextSegment(literalOffset: Offset) =
1452-
segmentBuf += Thicket(
1453-
literal(literalOffset, inPattern = inPattern, inStringInterpolation = true),
1454-
atSpan(in.offset) {
1455-
if (in.token == IDENTIFIER)
1456-
termIdent()
1457-
else if (in.token == USCORE && inPattern) {
1458-
in.nextToken()
1459-
Ident(nme.WILDCARD)
1460-
}
1461-
else if (in.token == THIS) {
1462-
in.nextToken()
1463-
This(EmptyTypeIdent)
1464-
}
1465-
else if (in.token == LBRACE)
1466-
if (inPattern) Block(Nil, inBraces(pattern()))
1467-
else expr()
1468-
else {
1469-
report.error(InterpolatedStringError(), source.atSpan(Span(in.offset)))
1470-
EmptyTree
1471-
}
1472-
})
14731514

1474-
var offsetCorrection = if isTripleQuoted then 3 else 1
1475-
while (in.token == STRINGPART)
1476-
nextSegment(in.offset + offsetCorrection)
1477-
offsetCorrection = 0
1478-
if (in.token == STRINGLIT)
1479-
segmentBuf += literal(inPattern = inPattern, negOffset = in.offset + offsetCorrection, inStringInterpolation = true)
1515+
// For dedented strings, we need to collect all string parts first,
1516+
// then dedent them all based on the closing indentation
1517+
if (isDedented) {
1518+
// Collect all string parts and their offsets
1519+
val stringParts = new ListBuffer[(String, Offset)]
1520+
val interpolatedExprs = new ListBuffer[Tree]
1521+
1522+
var offsetCorrection = 3 // triple single quotes
1523+
while (in.token == STRINGPART) {
1524+
val literalOffset = in.offset + offsetCorrection
1525+
stringParts += ((in.strVal, literalOffset))
1526+
offsetCorrection = 0
1527+
in.nextToken()
1528+
1529+
// Collect the interpolated expression
1530+
interpolatedExprs += atSpan(in.offset) {
1531+
if (in.token == IDENTIFIER)
1532+
termIdent()
1533+
else if (in.token == USCORE && inPattern) {
1534+
in.nextToken()
1535+
Ident(nme.WILDCARD)
1536+
}
1537+
else if (in.token == THIS) {
1538+
in.nextToken()
1539+
This(EmptyTypeIdent)
1540+
}
1541+
else if (in.token == LBRACE)
1542+
if (inPattern) Block(Nil, inBraces(pattern()))
1543+
else expr()
1544+
else {
1545+
report.error(InterpolatedStringError(), source.atSpan(Span(in.offset)))
1546+
EmptyTree
1547+
}
1548+
}
1549+
}
1550+
1551+
// Get the final STRINGLIT
1552+
val finalLiteral = if (in.token == STRINGLIT) {
1553+
val s = in.strVal
1554+
val off = in.offset + offsetCorrection
1555+
stringParts += ((s, off))
1556+
in.nextToken()
1557+
true
1558+
} else false
1559+
1560+
// Now dedent all string parts based on the last one's closing indentation
1561+
if (stringParts.nonEmpty) {
1562+
val lastPart = stringParts.last._1
1563+
val closingIndent = extractClosingIndent(lastPart)
1564+
1565+
// Dedent all parts
1566+
val dedentedParts = stringParts.map { case (str, offset) =>
1567+
(dedentStringPart(str, closingIndent), offset)
1568+
}
1569+
1570+
// Build the segments with dedented strings
1571+
for (i <- 0 until dedentedParts.size - 1) {
1572+
val (dedentedStr, offset) = dedentedParts(i)
1573+
segmentBuf += Thicket(
1574+
atSpan(offset, offset, offset + dedentedStr.length) { Literal(Constant(dedentedStr)) },
1575+
interpolatedExprs(i)
1576+
)
1577+
}
1578+
1579+
// Add the final literal if present
1580+
if (finalLiteral) {
1581+
val (dedentedStr, offset) = dedentedParts.last
1582+
segmentBuf += atSpan(offset, offset, offset + dedentedStr.length) { Literal(Constant(dedentedStr)) }
1583+
}
1584+
}
1585+
} else {
1586+
// Non-dedented string: use original logic
1587+
def nextSegment(literalOffset: Offset) =
1588+
segmentBuf += Thicket(
1589+
literal(literalOffset, inPattern = inPattern, inStringInterpolation = true),
1590+
atSpan(in.offset) {
1591+
if (in.token == IDENTIFIER)
1592+
termIdent()
1593+
else if (in.token == USCORE && inPattern) {
1594+
in.nextToken()
1595+
Ident(nme.WILDCARD)
1596+
}
1597+
else if (in.token == THIS) {
1598+
in.nextToken()
1599+
This(EmptyTypeIdent)
1600+
}
1601+
else if (in.token == LBRACE)
1602+
if (inPattern) Block(Nil, inBraces(pattern()))
1603+
else expr()
1604+
else {
1605+
report.error(InterpolatedStringError(), source.atSpan(Span(in.offset)))
1606+
EmptyTree
1607+
}
1608+
})
1609+
1610+
var offsetCorrection = if isTripleQuoted then 3 else 1
1611+
while (in.token == STRINGPART)
1612+
nextSegment(in.offset + offsetCorrection)
1613+
offsetCorrection = 0
1614+
if (in.token == STRINGLIT)
1615+
segmentBuf += literal(inPattern = inPattern, negOffset = in.offset + offsetCorrection, inStringInterpolation = true)
1616+
}
14801617

14811618
InterpolatedString(interpolator, segmentBuf.toList)
14821619
}
14831620

1621+
/** Extract the closing indentation from the last line of a string */
1622+
private def extractClosingIndent(str: String): String = {
1623+
val lastNewlineIdx = str.lastIndexOf('\n')
1624+
if (lastNewlineIdx < 0) "" else str.substring(lastNewlineIdx + 1)
1625+
}
1626+
1627+
/** Dedent a string part by removing the specified indentation from each line */
1628+
private def dedentStringPart(str: String, closingIndent: String): String = {
1629+
if (str.isEmpty || closingIndent.isEmpty) return str
1630+
1631+
val lines = str.split("\n", -1) // -1 to keep trailing empty strings
1632+
1633+
val dedented = lines.map { line =>
1634+
if (line.startsWith(closingIndent)) {
1635+
line.substring(closingIndent.length)
1636+
} else if (line.trim.isEmpty) {
1637+
// Empty or whitespace-only lines
1638+
""
1639+
} else {
1640+
// Line doesn't start with the closing indentation, keep as-is
1641+
line
1642+
}
1643+
}
1644+
1645+
dedented.mkString("\n")
1646+
}
1647+
14841648
/* ------------- NEW LINES ------------------------------------------------- */
14851649

14861650
def newLineOpt(): Unit =

‎compiler/src/dotty/tools/dotc/parsing/Scanners.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,10 +1349,12 @@ object Scanners {
13491349
foundQuotes += 1
13501350
nextRawChar()
13511351
}
1352+
charOffset -= 1
13521353

13531354
if (foundQuotes == quoteCount && ch != '\'') {
13541355
// Found closing delimiter - exact match and not followed by another quote
13551356
setStrVal()
1357+
nextChar() // Switch from raw mode to normal mode
13561358
token = STRINGLIT
13571359
} else {
13581360
// Not the closing delimiter, add the quotes we found to content

0 commit comments

Comments
 (0)