Skip to content

Commit 07ff5dc

Browse files
LuckyFBBLuckyFBB
authored
feat: unify variables in lexer (#366)
* feat: unify variables in lexer * fix: all sql use WHITE_SPACE
1 parent 042477d commit 07ff5dc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2478
-2511
lines changed

‎src/grammar/flink/FlinkSqlLexer.g4‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ options {
1111

1212
// SKIP
1313

14-
SPACE : [ \t\r\n]+ -> channel(HIDDEN);
15-
COMMENT_INPUT : '/*' .*? '*/' -> channel(HIDDEN);
14+
WHITE_SPACE : [ \t\r\n]+ -> channel(HIDDEN);
15+
BRACKETED_COMMENT : '/*' .*? '*/' -> channel(HIDDEN);
1616
LINE_COMMENT: (('--' | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);
1717

1818
/**

‎src/grammar/impala/ImpalaSqlLexer.g4‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -334,20 +334,20 @@ QUOTED_IDENTIFIER: '"' ( ~'"' | '""')* '"';
334334

335335
BACKQUOTED_IDENTIFIER: '`' ( ~'`' | '``')* '`';
336336

337-
TIME_WITH_TIME_ZONE: 'TIME' WS 'WITH' WS 'TIME' WS 'ZONE';
337+
TIME_WITH_TIME_ZONE: 'TIME' WHITE_SPACE 'WITH' WHITE_SPACE 'TIME' WHITE_SPACE 'ZONE';
338338

339-
TIMESTAMP_WITH_TIME_ZONE: 'TIMESTAMP' WS 'WITH' WS 'TIME' WS 'ZONE';
339+
TIMESTAMP_WITH_TIME_ZONE: 'TIMESTAMP' WHITE_SPACE 'WITH' WHITE_SPACE 'TIME' WHITE_SPACE 'ZONE';
340340

341-
DOUBLE_PRECISION: 'DOUBLE' WS 'PRECISION';
341+
DOUBLE_PRECISION: 'DOUBLE' WHITE_SPACE 'PRECISION';
342342

343343
fragment EXPONENT: 'E' [+-]? DIGIT+;
344344

345345
fragment DIGIT: [0-9];
346346

347347
fragment LETTER: [A-Z];
348348

349-
SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
349+
LINE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
350350

351351
BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
352352

353-
WS: [ \r\n\t]+ -> channel(HIDDEN);
353+
WHITE_SPACE: [ \r\n\t]+ -> channel(HIDDEN);

‎src/grammar/mysql/MySqlLexer.g4‎

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,23 @@
11
/*
2-
MySQL (Positive Technologies) grammar
3-
The MIT License (MIT).
4-
Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies.
5-
Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com)
6-
7-
Permission is hereby granted, free of charge, to any person obtaining a copy
8-
of this software and associated documentation files (the "Software"), to deal
9-
in the Software without restriction, including without limitation the rights
10-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11-
copies of the Software, and to permit persons to whom the Software is
12-
furnished to do so, subject to the following conditions:
13-
14-
The above copyright notice and this permission notice shall be included in
15-
all copies or substantial portions of the Software.
16-
17-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23-
THE SOFTWARE.
24-
*/
2+
MySQL (Positive Technologies) grammar The MIT License (MIT). Copyright (c) 2015-2017, Ivan
3+
Kochurkin (kvanttt@gmail.com), Positive Technologies. Copyright (c) 2017, Ivan Khudyashev
4+
(IHudyashov@ptsecurity.com)
5+
6+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
7+
associated documentation files (the "Software"), to deal in the Software without restriction,
8+
including without limitation the rights to use, copy, modify, merge, publish, distribute,
9+
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all copies or
13+
substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
16+
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18+
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
19+
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
20+
*/
2521

2622
// https://github.com/antlr/grammars-v4/blob/master/sql/mysql/Positive-Technologies/MySqlLexer.g4
2723

@@ -41,9 +37,9 @@ channels {
4137
ERRORCHANNEL
4238
}
4339

44-
SPACE : [ \t\r\n]+ -> channel(HIDDEN);
40+
WHITE_SPACE : [ \t\r\n]+ -> channel(HIDDEN);
4541
SPEC_MYSQL_COMMENT : '/*!' .+? '*/' -> channel(MYSQLCOMMENT);
46-
COMMENT_INPUT : '/*' .*? '*/' -> channel(HIDDEN);
42+
BRACKETED_COMMENT : '/*' .*? '*/' -> channel(HIDDEN);
4743
LINE_COMMENT: (('--' [ \t]* | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);
4844

4945
// Keywords

‎src/grammar/postgresql/PostgreSqlLexer.g4‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -804,22 +804,19 @@ PLSQLIDENTIFIER: ':"' ('\\' . | '""' | ~ ('"' | '\\'))* '"';
804804

805805
//
806806

807-
Whitespace: [ \t]+ -> channel (HIDDEN);
808-
809-
Newline: ('\r' '\n'? | '\n') -> channel (HIDDEN);
810-
//
807+
WHITE_SPACE: (' ' | '\r' | '\t' | '\n') -> channel(HIDDEN);
811808

812809
// COMMENTS (4.1.5)
813810

814811
//
815812

816-
LineComment: '--' ~ [\r\n]* -> channel (HIDDEN);
813+
LINE_COMMENT: '--' ~ [\r\n]* -> channel (HIDDEN);
817814

818-
BlockComment: ('/*' ( '/'* BlockComment | ~ [/*] | '/'+ ~ [/*] | '*'+ ~ [/*])* '*'* '*/') -> channel (HIDDEN);
815+
BRACKETED_COMMENT: ('/*' ( '/'* BRACKETED_COMMENT | ~ [/*] | '/'+ ~ [/*] | '*'+ ~ [/*])* '*'* '*/') -> channel (HIDDEN);
819816
820817
UnterminatedBlockComment:
821818
'/*' (
822-
'/'* BlockComment
819+
'/'* BRACKETED_COMMENT
823820
| // these characters are not part of special sequences in a block comment
824821
~ [/*]
825822
|
@@ -886,18 +883,14 @@ InvalidUnterminatedEscapeStringConstant:
886883
fragment InvalidEscapeStringText: ('\'\'' | '\\' . | ~ ['\\])*;
887884
888885
mode AfterEscapeStringConstantMode;
889-
AfterEscapeStringConstantMode_Whitespace: Whitespace -> type (Whitespace), channel (HIDDEN);
886+
AfterEscapeStringConstantMode_Whitespace: WHITE_SPACE -> type (WHITE_SPACE), channel (HIDDEN);
890887
891-
AfterEscapeStringConstantMode_Newline:
892-
Newline -> type (Newline), channel (HIDDEN), mode (AfterEscapeStringConstantWithNewlineMode);
893888
//
894889
// AfterEscapeStringConstantMode_NotContinued: '\\' ~[\r\n] -> skip, popMode;
895890
896891
mode AfterEscapeStringConstantWithNewlineMode;
897892
AfterEscapeStringConstantWithNewlineMode_Whitespace:
898-
Whitespace -> type (Whitespace), channel (HIDDEN);
899-
900-
AfterEscapeStringConstantWithNewlineMode_Newline: Newline -> type (Newline), channel (HIDDEN);
893+
WHITE_SPACE -> type (WHITE_SPACE), channel (HIDDEN);
901894
902895
AfterEscapeStringConstantWithNewlineMode_Continued:
903896
'\'' -> more, mode (EscapeStringConstantMode);

‎src/grammar/spark/SparkSqlLexer.g4‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -476,12 +476,12 @@ fragment DIGIT: [0-9];
476476

477477
fragment LETTER: [A-Z];
478478

479-
SIMPLE_COMMENT: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN);
479+
LINE_COMMENT: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN);
480480

481481
BRACKETED_COMMENT:
482482
'/*' (BRACKETED_COMMENT | .)*? ('*/' | {this.markUnclosedComment();} EOF) -> channel(HIDDEN);
483483

484-
WS: [ \r\n\t]+ -> channel(HIDDEN);
484+
WHITE_SPACE: [ \r\n\t]+ -> channel(HIDDEN);
485485

486486
// Catch-all for anything we can't recognize.
487487
// We use this to be able to ignore and recover all the text

‎src/grammar/trino/TrinoSql.g4‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1644,11 +1644,11 @@ fragment DIGIT: [0-9];
16441644

16451645
fragment LETTER: [A-Z];
16461646

1647-
SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
1647+
LINE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
16481648

16491649
BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
16501650

1651-
WS: [ \r\n\t]+ -> channel(HIDDEN);
1651+
WHITE_SPACE: [ \r\n\t]+ -> channel(HIDDEN);
16521652

16531653
// Catch-all for anything we can't recognize.
16541654
// We use this to be able to ignore and recover all the text

‎src/lib/flink/FlinkSqlLexer.interp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -544,8 +544,8 @@ null
544544

545545
token symbolic names:
546546
null
547-
SPACE
548-
COMMENT_INPUT
547+
WHITE_SPACE
548+
BRACKETED_COMMENT
549549
LINE_COMMENT
550550
KW_ABS
551551
KW_ALL
@@ -1087,8 +1087,8 @@ BIT_STRING
10871087
ID_LITERAL
10881088

10891089
rule names:
1090-
SPACE
1091-
COMMENT_INPUT
1090+
WHITE_SPACE
1091+
BRACKETED_COMMENT
10921092
LINE_COMMENT
10931093
KW_ABS
10941094
KW_ALL

‎src/lib/flink/FlinkSqlLexer.tokens‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
SPACE=1
2-
COMMENT_INPUT=2
1+
WHITE_SPACE=1
2+
BRACKETED_COMMENT=2
33
LINE_COMMENT=3
44
KW_ABS=4
55
KW_ALL=5

‎src/lib/flink/FlinkSqlLexer.ts‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import { Token } from "antlr4ng";
77

88

99
export class FlinkSqlLexer extends antlr.Lexer {
10-
public static readonly SPACE = 1;
11-
public static readonly COMMENT_INPUT = 2;
10+
public static readonly WHITE_SPACE = 1;
11+
public static readonly BRACKETED_COMMENT = 2;
1212
public static readonly LINE_COMMENT = 3;
1313
public static readonly KW_ABS = 4;
1414
public static readonly KW_ALL = 5;
@@ -655,21 +655,22 @@ export class FlinkSqlLexer extends antlr.Lexer {
655655
];
656656

657657
public static readonly symbolicNames = [
658-
null, "SPACE", "COMMENT_INPUT", "LINE_COMMENT", "KW_ABS", "KW_ALL",
659-
"KW_ALLOCATE", "KW_ALLOW", "KW_ALTER", "KW_ANALYZE", "KW_AND", "KW_ANY",
660-
"KW_ARE", "KW_ARRAY", "KW_ARRAY_AGG", "KW_ARRAY_CONCAT_AGG", "KW_ARRAY_MAX_CARDINALITY",
661-
"KW_AS", "KW_ASENSITIVE", "KW_ASYMMETRIC", "KW_AT", "KW_ATOMIC",
662-
"KW_AUTHORIZATION", "KW_AVG", "KW_BEGIN", "KW_BEGIN_FRAME", "KW_BEGIN_PARTITION",
663-
"KW_BETWEEN", "KW_BIGINT", "KW_BINARY", "KW_BIT", "KW_BLOB", "KW_BOOLEAN",
664-
"KW_BOTH", "KW_BY", "KW_BYTES", "KW_CALL", "KW_CALLED", "KW_CARDINALITY",
665-
"KW_CASCADED", "KW_CASE", "KW_CAST", "KW_CATALOGS", "KW_CEIL", "KW_CEILING",
666-
"KW_CHANGELOG_MODE", "KW_CHAR", "KW_CHARACTER", "KW_CHARACTER_LENGTH",
667-
"KW_CHAR_LENGTH", "KW_CHECK", "KW_CLASSIFIER", "KW_CLOB", "KW_CLOSE",
668-
"KW_COALESCE", "KW_COLLATE", "KW_COLLECT", "KW_COLUMN", "KW_COLUMNS",
669-
"KW_COMMENT", "KW_COMMIT", "KW_COMPUTE", "KW_CONDITION", "KW_CONNECT",
670-
"KW_CONSTRAINT", "KW_CONTAINS", "KW_CONVERT", "KW_CORR", "KW_CORRESPONDING",
671-
"KW_COUNT", "KW_COVAR_POP", "KW_COVAR_SAMP", "KW_CREATE", "KW_CROSS",
672-
"KW_CUBE", "KW_CUME_DIST", "KW_CURRENT", "KW_CURRENT_CATALOG", "KW_CURRENT_DATE",
658+
null, "WHITE_SPACE", "BRACKETED_COMMENT", "LINE_COMMENT", "KW_ABS",
659+
"KW_ALL", "KW_ALLOCATE", "KW_ALLOW", "KW_ALTER", "KW_ANALYZE", "KW_AND",
660+
"KW_ANY", "KW_ARE", "KW_ARRAY", "KW_ARRAY_AGG", "KW_ARRAY_CONCAT_AGG",
661+
"KW_ARRAY_MAX_CARDINALITY", "KW_AS", "KW_ASENSITIVE", "KW_ASYMMETRIC",
662+
"KW_AT", "KW_ATOMIC", "KW_AUTHORIZATION", "KW_AVG", "KW_BEGIN",
663+
"KW_BEGIN_FRAME", "KW_BEGIN_PARTITION", "KW_BETWEEN", "KW_BIGINT",
664+
"KW_BINARY", "KW_BIT", "KW_BLOB", "KW_BOOLEAN", "KW_BOTH", "KW_BY",
665+
"KW_BYTES", "KW_CALL", "KW_CALLED", "KW_CARDINALITY", "KW_CASCADED",
666+
"KW_CASE", "KW_CAST", "KW_CATALOGS", "KW_CEIL", "KW_CEILING", "KW_CHANGELOG_MODE",
667+
"KW_CHAR", "KW_CHARACTER", "KW_CHARACTER_LENGTH", "KW_CHAR_LENGTH",
668+
"KW_CHECK", "KW_CLASSIFIER", "KW_CLOB", "KW_CLOSE", "KW_COALESCE",
669+
"KW_COLLATE", "KW_COLLECT", "KW_COLUMN", "KW_COLUMNS", "KW_COMMENT",
670+
"KW_COMMIT", "KW_COMPUTE", "KW_CONDITION", "KW_CONNECT", "KW_CONSTRAINT",
671+
"KW_CONTAINS", "KW_CONVERT", "KW_CORR", "KW_CORRESPONDING", "KW_COUNT",
672+
"KW_COVAR_POP", "KW_COVAR_SAMP", "KW_CREATE", "KW_CROSS", "KW_CUBE",
673+
"KW_CUME_DIST", "KW_CURRENT", "KW_CURRENT_CATALOG", "KW_CURRENT_DATE",
673674
"KW_CURRENT_DEFAULT_TRANSFORM_GROUP", "KW_CURRENT_PATH", "KW_CURRENT_ROLE",
674675
"KW_CURRENT_ROW", "KW_CURRENT_SCHEMA", "KW_CURRENT_TIME", "KW_CURRENT_TIMESTAMP",
675676
"KW_CURRENT_TRANSFORM_GROUP_FOR_TYPE", "KW_CURRENT_USER", "KW_CURSOR",
@@ -770,9 +771,9 @@ export class FlinkSqlLexer extends antlr.Lexer {
770771
];
771772

772773
public static readonly ruleNames = [
773-
"SPACE", "COMMENT_INPUT", "LINE_COMMENT", "KW_ABS", "KW_ALL", "KW_ALLOCATE",
774-
"KW_ALLOW", "KW_ALTER", "KW_ANALYZE", "KW_AND", "KW_ANY", "KW_ARE",
775-
"KW_ARRAY", "KW_ARRAY_AGG", "KW_ARRAY_CONCAT_AGG", "KW_ARRAY_MAX_CARDINALITY",
774+
"WHITE_SPACE", "BRACKETED_COMMENT", "LINE_COMMENT", "KW_ABS", "KW_ALL",
775+
"KW_ALLOCATE", "KW_ALLOW", "KW_ALTER", "KW_ANALYZE", "KW_AND", "KW_ANY",
776+
"KW_ARE", "KW_ARRAY", "KW_ARRAY_AGG", "KW_ARRAY_CONCAT_AGG", "KW_ARRAY_MAX_CARDINALITY",
776777
"KW_AS", "KW_ASENSITIVE", "KW_ASYMMETRIC", "KW_AT", "KW_ATOMIC",
777778
"KW_AUTHORIZATION", "KW_AVG", "KW_BEGIN", "KW_BEGIN_FRAME", "KW_BEGIN_PARTITION",
778779
"KW_BETWEEN", "KW_BIGINT", "KW_BINARY", "KW_BIT", "KW_BLOB", "KW_BOOLEAN",

‎src/lib/flink/FlinkSqlParser.interp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -544,8 +544,8 @@ null
544544

545545
token symbolic names:
546546
null
547-
SPACE
548-
COMMENT_INPUT
547+
WHITE_SPACE
548+
BRACKETED_COMMENT
549549
LINE_COMMENT
550550
KW_ABS
551551
KW_ALL

0 commit comments

Comments
 (0)