Skip to content

Commit 9eb87a2

Browse files
lihaoyilihaoyi
committed
.
1 parent b9f6f2e commit 9eb87a2

File tree

7 files changed

+508
-1
lines changed

7 files changed

+508
-1
lines changed

‎compiler/src/dotty/tools/dotc/parsing/Scanners.scala‎

Lines changed: 167 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,19 @@ object Scanners {
978978
case '\'' =>
979979
def fetchSingleQuote(): Unit = {
980980
nextChar()
981-
if isIdentifierStart(ch) then
981+
// Check for triple single quote (dedented string literal)
982+
if (ch == '\'') {
983+
nextChar()
984+
if (ch == '\'') {
985+
// We have at least '''
986+
getDedentedStringLit()
987+
}
988+
else {
989+
// We have '' followed by something else
990+
error(em"empty character literal")
991+
}
992+
}
993+
else if isIdentifierStart(ch) then
982994
charLitOr { getIdentRest(); QUOTEID }
983995
else if isOperatorPart(ch) && ch != '\\' then
984996
charLitOr { getOperatorRest(); QUOTEID }
@@ -1255,6 +1267,160 @@ object Scanners {
12551267
else error(em"unclosed string literal")
12561268
}
12571269

1270+
/** Parse a dedented string literal (triple single quotes)
1271+
* Requirements:
1272+
* - Must start with ''' followed by newline
1273+
* - Must end with newline + whitespace + '''
1274+
* - Removes first newline after opening delimiter
1275+
* - Removes final newline and preceding whitespace before closing delimiter
1276+
* - Strips indentation equal to closing delimiter indentation
1277+
* - All lines must be empty or indented further than closing delimiter
1278+
* - Supports extended delimiters (e.g., '''', ''''')
1279+
*/
1280+
private def getDedentedStringLit(): Unit = {
1281+
// Count opening quotes (already consumed 3)
1282+
nextChar()
1283+
var quoteCount = 3
1284+
while (ch == '\'') {
1285+
quoteCount += 1
1286+
nextChar()
1287+
}
1288+
1289+
// Must be followed by a newline
1290+
if (ch != LF && ch != CR) {
1291+
error(em"dedented string literal must start with newline after opening quotes")
1292+
token = ERROR
1293+
} else {
1294+
// Skip the initial newline (CR LF or just LF)
1295+
if (ch == CR) nextRawChar()
1296+
if (ch == LF) nextRawChar()
1297+
1298+
// Collect all lines until we find the closing delimiter
1299+
val lines = scala.collection.mutable.ArrayBuffer[String]()
1300+
val lineIndents = scala.collection.mutable.ArrayBuffer[String]()
1301+
var currentLine = new StringBuilder
1302+
var currentIndent = new StringBuilder
1303+
var atLineStart = true
1304+
var closingIndent: String = null
1305+
var foundClosing = false
1306+
1307+
while (!foundClosing && ch != SU) {
1308+
if (atLineStart) {
1309+
// Collect indentation
1310+
currentIndent.clear()
1311+
while (ch == ' ' || ch == '\t') {
1312+
currentIndent.append(ch)
1313+
nextRawChar()
1314+
}
1315+
1316+
// Check if this might be the closing delimiter
1317+
if (ch == '\'') {
1318+
var endQuoteCount = 0
1319+
val savedOffset = charOffset
1320+
while (ch == '\'' && endQuoteCount < quoteCount + 1) {
1321+
endQuoteCount += 1
1322+
nextRawChar()
1323+
}
1324+
1325+
if (endQuoteCount == quoteCount && (ch == SU || ch == CR || ch == LF || ch == ' ' || ch == '\t' || ch == ';')) {
1326+
// Found closing delimiter
1327+
foundClosing = true
1328+
closingIndent = currentIndent.toString
1329+
// Consume any trailing whitespace/newlines after closing quotes
1330+
while (ch == ' ' || ch == '\t') nextChar()
1331+
if (ch == CR || ch == LF) nextChar()
1332+
} else {
1333+
// False alarm, these quotes are part of the content
1334+
// We need to restore and add them to current line
1335+
currentLine.append(currentIndent)
1336+
for (_ <- 0 until endQuoteCount) currentLine.append('\'')
1337+
atLineStart = false
1338+
}
1339+
} else {
1340+
atLineStart = false
1341+
}
1342+
}
1343+
1344+
if (!foundClosing && !atLineStart) {
1345+
// Regular content
1346+
if (ch == CR || ch == LF) {
1347+
// End of line
1348+
lineIndents += currentIndent.toString
1349+
lines += currentLine.toString
1350+
currentLine.clear()
1351+
currentIndent.clear()
1352+
1353+
// Normalize newlines to \n
1354+
if (ch == CR) nextRawChar()
1355+
if (ch == LF) nextRawChar()
1356+
atLineStart = true
1357+
} else {
1358+
currentLine.append(ch)
1359+
nextRawChar()
1360+
}
1361+
}
1362+
}
1363+
1364+
if (!foundClosing) {
1365+
incompleteInputError(em"unclosed dedented string literal")
1366+
} else if (closingIndent == null) {
1367+
error(em"internal error: closing indent not set")
1368+
token = ERROR
1369+
} else {
1370+
// Validate and dedent all lines
1371+
val dedentedLines = scala.collection.mutable.ArrayBuffer[String]()
1372+
val closingIndentLen = closingIndent.length
1373+
var hasSpaces = false
1374+
var hasTabs = false
1375+
1376+
for (indent <- closingIndent) {
1377+
if (indent == ' ') hasSpaces = true
1378+
if (indent == '\t') hasTabs = true
1379+
}
1380+
1381+
var hasError = false
1382+
for (i <- 0 until lines.length if !hasError) {
1383+
val line = lines(i)
1384+
val indent = lineIndents(i)
1385+
1386+
// Check for mixed tabs and spaces
1387+
var lineHasSpaces = false
1388+
var lineHasTabs = false
1389+
for (ch <- indent) {
1390+
if (ch == ' ') lineHasSpaces = true
1391+
if (ch == '\t') lineHasTabs = true
1392+
}
1393+
1394+
if ((hasSpaces && lineHasTabs) || (hasTabs && lineHasSpaces)) {
1395+
error(em"dedented string literal cannot mix tabs and spaces in indentation")
1396+
token = ERROR
1397+
hasError = true
1398+
} else if (line.isEmpty) {
1399+
// Empty lines are allowed
1400+
dedentedLines += ""
1401+
} else {
1402+
// Non-empty lines must be indented at least as much as closing delimiter
1403+
if (!indent.startsWith(closingIndent)) {
1404+
error(em"line in dedented string literal must be indented at least as much as the closing delimiter")
1405+
token = ERROR
1406+
hasError = true
1407+
} else {
1408+
// Remove the closing indentation from this line
1409+
dedentedLines += indent.substring(closingIndentLen) + line
1410+
}
1411+
}
1412+
}
1413+
1414+
if (!hasError) {
1415+
// Set the string value (join with \n)
1416+
strVal = dedentedLines.mkString("\n")
1417+
litBuf.clear()
1418+
token = STRINGLIT
1419+
}
1420+
}
1421+
}
1422+
}
1423+
12581424
private def getRawStringLit(): Unit =
12591425
if (ch == '\"') {
12601426
nextRawChar()

‎project/project/build.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
sbt.version=1.10.2

‎tests/neg/dedented-string-literals.check‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
-- Error: tests/neg/dedented-string-literals.scala:5:36
2+
5 | val noNewlineAfterOpen = '''content on same line // error
3+
| ^
4+
|dedented string literal must start with newline after opening quotes
5+
-- Error: tests/neg/dedented-string-literals.scala:8:0
6+
8 |content
7+
|^
8+
|line in dedented string literal must be indented at least as much as the closing delimiter
9+
-- Error: tests/neg/dedented-string-literals.scala:14:0
10+
14 | space line
11+
| ^
12+
|dedented string literal cannot mix tabs and spaces in indentation
13+
-- [E040] Syntax Error: tests/neg/dedented-string-literals.scala:19:0
14+
19 | // error: missing closing quotes
15+
| ^^
16+
|unclosed dedented string literal

‎tests/neg/dedented-string-literals.scala‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Test error cases for dedented string literals
2+
3+
object DedentedStringErrors {
4+
// Error: No newline after opening quotes
5+
val noNewlineAfterOpen = '''content on same line // error
6+
7+
// Error: Content not indented enough
8+
val notIndented = '''
9+
content
10+
''' // error
11+
12+
// Error: Mixed tabs and spaces
13+
val mixedTabsSpaces = '''
14+
tab line
15+
space line
16+
''' // error
17+
18+
// Error: Unclosed literal
19+
val unclosed = '''
20+
some content
21+
// error: missing closing quotes
22+
}

‎tests/pos/dedented-string-literals.scala‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Test dedented string literals as specified in SIP
2+
3+
object DedentedStringLiterals {
4+
// Basic usage
5+
val basic = '''
6+
i am cow
7+
hear me moo
8+
'''
9+
10+
// With indentation preserved
11+
val withIndent = '''
12+
i am cow
13+
hear me moo
14+
'''
15+
16+
// Empty string
17+
val empty = '''
18+
'''
19+
20+
// Single line of content
21+
val singleLine = '''
22+
hello world
23+
'''
24+
25+
// Multiple blank lines
26+
val blankLines = '''
27+
line 1
28+
29+
line 3
30+
'''
31+
32+
// Deep indentation
33+
val deepIndent = '''
34+
deeply
35+
indented
36+
content
37+
'''
38+
39+
// Mixed content indentation (more than closing)
40+
val mixedIndent = '''
41+
first level
42+
second level
43+
third level
44+
'''
45+
46+
// Using extended delimiter to include '''
47+
val withTripleQuotes = ''''
48+
'''
49+
i am cow
50+
hear me moo
51+
'''
52+
''''
53+
54+
// Extended delimiter with 5 quotes
55+
val extended5 = '''''
56+
''''
57+
content with four quotes
58+
''''
59+
'''''
60+
61+
// Tabs for indentation
62+
val withTabs = '''
63+
tab indented
64+
content here
65+
'''
66+
67+
// Empty lines are allowed anywhere
68+
val emptyLinesAnywhere = '''
69+
70+
content
71+
72+
more content
73+
74+
'''
75+
76+
// Testing in different contexts
77+
def inFunction = '''
78+
function content
79+
more content
80+
'''
81+
82+
class InClass {
83+
val inClass = '''
84+
class member
85+
content
86+
'''
87+
}
88+
89+
// In a list
90+
val list = List(
91+
'''
92+
first
93+
''',
94+
'''
95+
second
96+
''',
97+
'''
98+
third
99+
'''
100+
)
101+
102+
// Nested in expressions
103+
val nested = "prefix" + '''
104+
middle
105+
''' + "suffix"
106+
107+
// With special characters
108+
val specialChars = '''
109+
!"#$%&()*+,-./:;<=>?@[\]^_`{|}~
110+
'''
111+
112+
// Unicode content
113+
val unicode = '''
114+
Hello 世界
115+
Καλημέρα κόσμε
116+
'''
117+
118+
// Zero-width closing indent
119+
val zeroIndent = '''
120+
content
121+
'''
122+
123+
// Content with quotes
124+
val withQuotes = '''
125+
"double quotes"
126+
'single quote'
127+
''
128+
'''
129+
}

0 commit comments

Comments
 (0)