Skip to content

Commit 0dd12da

Browse files
jakebaileyjakebailey
authored
Error on replacement character only in top-level scanning (#58227)
1 parent cbae6cf commit 0dd12da

File tree

9 files changed

+85
-15
lines changed

9 files changed

+85
-15
lines changed

‎src/compiler/scanner.ts‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1768,18 +1768,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
17681768

17691769
const ch = codePointAt(text, pos);
17701770
if (pos === 0) {
1771-
// If a file isn't valid text at all, it will usually be apparent
1772-
// in the first few characters because UTF-8 decode will fail and produce U+FFFD.
1773-
// If that happens, just issue one error and refuse to try to scan further;
1774-
// this is likely a binary file that cannot be parsed.
1775-
//
1776-
// It's safe to slice the text; U+FFFD can only be produced by an invalid decode,
1777-
// so even if we cut a surrogate pair in half, they wouldn't be U+FFFD.
1778-
if (text.slice(0, 256).includes("\uFFFD")) {
1779-
error(Diagnostics.File_appears_to_be_binary);
1780-
pos = end;
1781-
return token = SyntaxKind.NonTextFileMarkerTrivia;
1782-
}
17831771
// Special handling for shebang
17841772
if (ch === CharacterCodes.hash && isShebangTrivia(text, pos)) {
17851773
pos = scanShebangTrivia(text, pos);
@@ -2242,6 +2230,10 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
22422230
error(Diagnostics.Invalid_character, pos++, charSize(ch));
22432231
}
22442232
return token = SyntaxKind.PrivateIdentifier;
2233+
case CharacterCodes.replacementCharacter:
2234+
error(Diagnostics.File_appears_to_be_binary, 0, 0);
2235+
pos = end;
2236+
return token = SyntaxKind.NonTextFileMarkerTrivia;
22452237
default:
22462238
const identifierKind = scanIdentifier(ch, languageVersion);
22472239
if (identifierKind) {

‎src/compiler/types.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7612,6 +7612,9 @@ export const enum CharacterCodes {
76127612
mathematicalSpace = 0x205F,
76137613
ogham = 0x1680,
76147614

7615+
// Unicode replacement character produced when a byte sequence is invalid
7616+
replacementCharacter = 0xFFFD,
7617+
76157618
_ = 0x5F,
76167619
$ = 0x24,
76177620

‎tests/baselines/reference/TransportStream.errors.txt‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,19 @@
11
TransportStream.ts(1,1): error TS1490: File appears to be binary.
2+
TransportStream.ts(1,1): error TS1434: Unexpected keyword or identifier.
3+
TransportStream.ts(1,1): error TS2304: Cannot find name 'G'.
4+
TransportStream.ts(1,3): error TS1127: Invalid character.
5+
TransportStream.ts(1,4): error TS1128: Declaration or statement expected.
26

37

4-
==== TransportStream.ts (1 errors) ====
8+
==== TransportStream.ts (5 errors) ====
59
G@�G@�G@�
610

7-
!!! error TS1490: File appears to be binary.
11+
!!! error TS1490: File appears to be binary.
12+
~
13+
!!! error TS1434: Unexpected keyword or identifier.
14+
~
15+
!!! error TS2304: Cannot find name 'G'.
16+
~
17+
!!! error TS1127: Invalid character.
18+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
19+
!!! error TS1128: Declaration or statement expected.

‎tests/baselines/reference/TransportStream.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@
44
G@�G@�G@�
55

66
//// [TransportStream.js]
7+
G;

‎tests/baselines/reference/TransportStream.types‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
//// [tests/cases/compiler/TransportStream.ts] ////
22

33
=== TransportStream.ts ===
4-
54
G@�G@�G@�
5+
>G : any
6+
> : ^^^
7+
> : any
8+
> : ^^^
9+

‎tests/baselines/reference/parseReplacementCharacter.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
//// [tests/cases/compiler/parseReplacementCharacter.ts] ////
2+
3+
//// [parseReplacementCharacter.ts]
4+
"oops �� oops";
5+
'oops �� oops';
6+
`oops �� oops`;
7+
`${"oops �� oops"}`;
8+
// oops �� oops
9+
/* oops �� oops */
10+
/** oops �� oops */
11+
12+
//// [parseReplacementCharacter.js]
13+
"oops �� oops";
14+
'oops �� oops';
15+
"oops \uFFFD\uFFFD oops";
16+
"".concat("oops �� oops");

‎tests/baselines/reference/parseReplacementCharacter.symbols‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
//// [tests/cases/compiler/parseReplacementCharacter.ts] ////
2+
3+
=== parseReplacementCharacter.ts ===
4+
5+
"oops �� oops";
6+
'oops �� oops';
7+
`oops �� oops`;
8+
`${"oops �� oops"}`;
9+
// oops �� oops
10+
/* oops �� oops */
11+
/** oops �� oops */

‎tests/baselines/reference/parseReplacementCharacter.types‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
//// [tests/cases/compiler/parseReplacementCharacter.ts] ////
2+
3+
=== parseReplacementCharacter.ts ===
4+
"oops �� oops";
5+
>"oops �� oops" : "oops �� oops"
6+
> : ^^^^^^^^^^^^^^
7+
8+
'oops �� oops';
9+
>'oops �� oops' : "oops �� oops"
10+
> : ^^^^^^^^^^^^^^
11+
12+
`oops �� oops`;
13+
>`oops �� oops` : "oops �� oops"
14+
> : ^^^^^^^^^^^^^^
15+
16+
`${"oops �� oops"}`;
17+
>`${"oops �� oops"}` : "oops �� oops"
18+
> : ^^^^^^^^^^^^^^
19+
>"oops �� oops" : "oops �� oops"
20+
> : ^^^^^^^^^^^^^^
21+
22+
// oops �� oops
23+
/* oops �� oops */
24+
/** oops �� oops */

‎tests/cases/compiler/parseReplacementCharacter.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
"oops �� oops";
2+
'oops �� oops';
3+
`oops �� oops`;
4+
`${"oops �� oops"}`;
5+
// oops �� oops
6+
/* oops �� oops */
7+
/** oops �� oops */

0 commit comments

Comments
 (0)