Skip to content

Commit 2a6ec8e

Browse files
nicolasstuckinicolasstucki
authored
Merge pull request #6392 from Jentsch/6332-numeric-underscore
Accept underscore as numeric literal separator
2 parents 0cf7ce3 + 11d533a commit 2a6ec8e

File tree

6 files changed

+164
-28
lines changed

6 files changed

+164
-28
lines changed

‎compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ object JavaScanners {
521521
case FLOATLIT =>
522522
"float(" + floatVal + ")"
523523
case DOUBLELIT =>
524-
"double(" + floatVal + ")"
524+
"double(" + doubleVal + ")"
525525
case STRINGLIT =>
526526
"string(" + name + ")"
527527
case SEMI =>

‎compiler/src/dotty/tools/dotc/parsing/Parsers.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,7 @@ object Parsers {
726726
case INTLIT => in.intVal(isNegated).toInt
727727
case LONGLIT => in.intVal(isNegated)
728728
case FLOATLIT => in.floatVal(isNegated).toFloat
729-
case DOUBLELIT => in.floatVal(isNegated)
729+
case DOUBLELIT => in.doubleVal(isNegated)
730730
case STRINGLIT | STRINGPART => in.strVal
731731
case TRUE => true
732732
case FALSE => false

‎compiler/src/dotty/tools/dotc/parsing/Scanners.scala‎

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,15 @@ object Scanners {
6767

6868
/** Generate an error at the given offset */
6969
def error(msg: String, off: Offset = offset): Unit = {
70-
ctx.error(msg, source atSpan Span(off))
70+
errorButContinue(msg, off)
7171
token = ERROR
7272
errOffset = off
7373
}
7474

75+
def errorButContinue(msg: String, off: Offset = offset): Unit = {
76+
ctx.error(msg, source atSpan Span(off))
77+
}
78+
7579
/** signal an error where the input ended in the middle of a token */
7680
def incompleteInputError(msg: String): Unit = {
7781
ctx.incompleteInputError(msg, source atSpan Span(offset))
@@ -129,19 +133,22 @@ object Scanners {
129133
var i = 0
130134
val len = strVal.length
131135
while (i < len) {
132-
val d = digit2int(strVal charAt i, base)
133-
if (d < 0) {
134-
error("malformed integer number")
135-
return 0
136-
}
137-
if (value < 0 ||
138-
limit / (base / divider) < value ||
139-
limit - (d / divider) < value * (base / divider) &&
140-
!(negated && limit == value * base - 1 + d)) {
141-
error("integer number too large")
142-
return 0
136+
val c = strVal charAt i
137+
if (! isNumberSeparator(c)) {
138+
val d = digit2int(c, base)
139+
if (d < 0) {
140+
error(s"malformed integer number")
141+
return 0
142+
}
143+
if (value < 0 ||
144+
limit / (base / divider) < value ||
145+
limit - (d / divider) < value * (base / divider) &&
146+
!(negated && limit == value * base - 1 + d)) {
147+
error("integer number too large")
148+
return 0
149+
}
150+
value = value * base + d
143151
}
144-
value = value * base + d
145152
i += 1
146153
}
147154
if (negated) -value else value
@@ -150,15 +157,42 @@ object Scanners {
150157

151158
def intVal: Long = intVal(false)
152159

160+
private val zeroFloat = raw"[0.]+(?:[eE][+-]?[0-9]+)?[fFdD]?".r
161+
162+
/** Convert current strVal, base to double value
163+
*/
164+
def floatVal(negated: Boolean): Float = {
165+
assert(token == FLOATLIT)
166+
val text = removeNumberSeparators(strVal)
167+
try {
168+
val value: Float = java.lang.Float.valueOf(text).floatValue()
169+
if (value > Float.MaxValue)
170+
errorButContinue("floating point number too large")
171+
172+
if (value == 0.0f && !zeroFloat.pattern.matcher(text).matches)
173+
errorButContinue("floating point number too small")
174+
if (negated) -value else value
175+
} catch {
176+
case _: NumberFormatException =>
177+
error("malformed floating point number")
178+
0.0f
179+
}
180+
}
181+
182+
def floatVal: Float = floatVal(false)
183+
153184
/** Convert current strVal, base to double value
154185
*/
155-
def floatVal(negated: Boolean): Double = {
156-
val limit: Double =
157-
if (token == DOUBLELIT) Double.MaxValue else Float.MaxValue
186+
def doubleVal(negated: Boolean): Double = {
187+
assert(token == DOUBLELIT)
188+
val text = removeNumberSeparators(strVal)
158189
try {
159-
val value: Double = java.lang.Double.valueOf(strVal).doubleValue()
160-
if (value > limit)
161-
error("floating point number too large")
190+
val value: Double = java.lang.Double.valueOf(text).doubleValue()
191+
if (value > Double.MaxValue)
192+
errorButContinue("double precision floating point number too large")
193+
194+
if (value == 0.0d && !zeroFloat.pattern.matcher(text).matches)
195+
errorButContinue("double precision floating point number too small")
162196
if (negated) -value else value
163197
} catch {
164198
case _: NumberFormatException =>
@@ -167,7 +201,18 @@ object Scanners {
167201
}
168202
}
169203

170-
def floatVal: Double = floatVal(false)
204+
def doubleVal: Double = doubleVal(false)
205+
206+
@inline def isNumberSeparator(c: Char): Boolean = c == '_'
207+
208+
@inline def removeNumberSeparators(s: String): String =
209+
if (s.indexOf('_') > 0) s.replaceAllLiterally("_", "") /*.replaceAll("'","")*/ else s
210+
211+
// disallow trailing numeric separator char, but continue lexing
212+
def checkNoTrailingSeparator(): Unit = {
213+
if (isNumberSeparator(litBuf.last))
214+
errorButContinue("trailing separator is not allowed", offset + litBuf.length - 1)
215+
}
171216

172217
}
173218

@@ -463,6 +508,8 @@ object Scanners {
463508
if (ch == 'x' || ch == 'X') {
464509
nextChar()
465510
base = 16
511+
if (isNumberSeparator(ch))
512+
errorButContinue("leading separator is not allowed", offset + 2)
466513
} else {
467514
/**
468515
* What should leading 0 be in the future? It is potentially dangerous
@@ -911,27 +958,29 @@ object Scanners {
911958
*/
912959
protected def getFraction(): Unit = {
913960
token = DOUBLELIT
914-
while ('0' <= ch && ch <= '9') {
961+
while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
915962
putChar(ch)
916963
nextChar()
917964
}
965+
checkNoTrailingSeparator()
918966
if (ch == 'e' || ch == 'E') {
919967
val lookahead = lookaheadReader()
920968
lookahead.nextChar()
921969
if (lookahead.ch == '+' || lookahead.ch == '-') {
922970
lookahead.nextChar()
923971
}
924-
if ('0' <= lookahead.ch && lookahead.ch <= '9') {
972+
if ('0' <= lookahead.ch && lookahead.ch <= '9' || isNumberSeparator(ch)) {
925973
putChar(ch)
926974
nextChar()
927975
if (ch == '+' || ch == '-') {
928976
putChar(ch)
929977
nextChar()
930978
}
931-
while ('0' <= ch && ch <= '9') {
979+
while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
932980
putChar(ch)
933981
nextChar()
934982
}
983+
checkNoTrailingSeparator()
935984
}
936985
token = DOUBLELIT
937986
}
@@ -954,15 +1003,18 @@ object Scanners {
9541003
/** Read a number into strVal and set base
9551004
*/
9561005
protected def getNumber(): Unit = {
957-
while (digit2int(ch, base) >= 0) {
1006+
while (isNumberSeparator(ch) || digit2int(ch, base) >= 0) {
9581007
putChar(ch)
9591008
nextChar()
9601009
}
1010+
checkNoTrailingSeparator()
9611011
token = INTLIT
9621012
if (base == 10 && ch == '.') {
9631013
val lch = lookaheadChar()
9641014
if ('0' <= lch && lch <= '9') {
965-
putChar('.'); nextChar(); getFraction()
1015+
putChar('.')
1016+
nextChar()
1017+
getFraction()
9661018
}
9671019
} else (ch: @switch) match {
9681020
case 'e' | 'E' | 'f' | 'F' | 'd' | 'D' =>
@@ -972,6 +1024,9 @@ object Scanners {
9721024
token = LONGLIT
9731025
case _ =>
9741026
}
1027+
1028+
checkNoTrailingSeparator()
1029+
9751030
setStrVal()
9761031
}
9771032

@@ -1007,7 +1062,7 @@ object Scanners {
10071062
case INTLIT => s"int($intVal)"
10081063
case LONGLIT => s"long($intVal)"
10091064
case FLOATLIT => s"float($floatVal)"
1010-
case DOUBLELIT => s"double($floatVal)"
1065+
case DOUBLELIT => s"double($doubleVal)"
10111066
case STRINGLIT => s"string($strVal)"
10121067
case STRINGPART => s"stringpart($strVal)"
10131068
case INTERPOLATIONID => s"interpolationid($name)"

‎tests/neg/t6124.check‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[490..493] in t6124.scala
2+
Not found: _52
3+
[404..406..412] in t6124.scala
4+
value _1415F is not a member of Int
5+
<594..594> in t6124.scala
6+
trailing separator is not allowed
7+
<567..567> in t6124.scala
8+
leading separator is not allowed
9+
<540..540> in t6124.scala
10+
trailing separator is not allowed
11+
<516..516> in t6124.scala
12+
trailing separator is not allowed
13+
<467..467> in t6124.scala
14+
trailing separator is not allowed
15+
<375..375> in t6124.scala
16+
trailing separator is not allowed
17+
<228..228> in t6124.scala
18+
double precision floating point number too small
19+
<197..197> in t6124.scala
20+
trailing separator is not allowed
21+
<159..159> in t6124.scala
22+
Invalid literal number
23+
<164..164> in t6124.scala
24+
trailing separator is not allowed
25+
<137..137> in t6124.scala
26+
trailing separator is not allowed
27+
<111..111> in t6124.scala
28+
trailing separator is not allowed
29+
<84..84> in t6124.scala
30+
trailing separator is not allowed
31+
<49..49> in t6124.scala
32+
Invalid literal number
33+
<25..25> in t6124.scala
34+
trailing separator is not allowed

‎tests/neg/t6124.scala‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
2+
trait T {
3+
def f = 3_14_E-2 // error
4+
def e = 3_14E-_2 // error
5+
def d = 3_14E-2_ // error
6+
7+
def p = 3.1_4_ // error
8+
def q = 3.1_4_d // error
9+
def r = 3.1_4_dd // error // error
10+
def s = 3_.1 // error
11+
12+
def tooSmall = 1.0E-325 // error
13+
14+
// Examples from
15+
// https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html
16+
17+
val pi1 = 3_.1415F // error
18+
val pi2 = 3._1415F // error
19+
val socialSecurityNumber1
20+
= 999_99_9999_L // error
21+
val x1 = _52 // error
22+
val x3 = 52_ // error
23+
24+
val x5 = 0_x52 // error
25+
val x6 = 0x_52 // error
26+
val x8 = 0x52_ // error
27+
28+
def z = 0
29+
}

‎tests/pos/t6124.scala‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
2+
trait T {
3+
def i: Int = 1_024
4+
def j: Long = 1_024L * 1_024
5+
//def k = 1'024
6+
7+
def f = 3_14e-2f
8+
def d = 3_14E-2_1
9+
10+
def z = 0
11+
12+
13+
// Examples from
14+
// https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html
15+
val x2 = 5_2; // OK (decimal literal)
16+
val x4 = 5_______2; // OK (decimal literal)
17+
val x7 = 0x5_2; // OK (hexadecimal literal)
18+
}

0 commit comments

Comments
 (0)