Skip to content

Commit f131d9a

Browse files
maxbrunsfeldmaxbrunsfeld
committed
Remove redundant characters in non_special_token
1 parent 2a5c1ff commit f131d9a

File tree

4 files changed

+43
-28
lines changed

4 files changed

+43
-28
lines changed

‎corpus/macros.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,22 @@ a!('lifetime)
5454
(identifier)
5555
(token_tree (identifier))))
5656

57+
=====================================
58+
Macro invocation with comments
59+
=====================================
60+
61+
ok! {
62+
// one
63+
/* two */
64+
}
65+
66+
---
67+
68+
(source_file
69+
(macro_invocation
70+
(identifier)
71+
(token_tree (line_comment) (block_comment))))
72+
5773
============================================
5874
Macro definition
5975
============================================

‎grammar.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ module.exports = grammar({
190190
_non_special_token: $ => choice(
191191
$._literal, $.identifier, $.metavariable, $.mutable_specifier, $.self, $.super, $.crate,
192192
alias(choice(...primitive_types), $.primitive_type),
193-
/[/_\-=->,;:::!=?.@*=/=&=#%=^=+<>|~]+/,
193+
/[/_\-=->,;:::!=?.@*&#%^+<>|~]+/,
194194
'\'',
195195
'as', 'async', 'await', 'break', 'const', 'continue', 'default', 'enum', 'fn', 'for', 'if', 'impl',
196196
'let', 'loop', 'match', 'mod', 'pub', 'return', 'static', 'struct', 'trait', 'type',

‎src/grammar.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -755,7 +755,7 @@
755755
},
756756
{
757757
"type": "PATTERN",
758-
"value": "[/_\\-=->,;:::!=?.@*=/=&=#%=^=+<>|~]+"
758+
"value": "[/_\\-=->,;:::!=?.@*&#%^+<>|~]+"
759759
},
760760
{
761761
"type": "STRING",

‎src/parser.c‎

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6926,12 +6926,12 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
69266926
if (lookahead == '\'') ADVANCE(86);
69276927
if (lookahead == '(') ADVANCE(61);
69286928
if (lookahead == ')') ADVANCE(62);
6929-
if (lookahead == '/') ADVANCE(82);
6929+
if (lookahead == '/') ADVANCE(81);
69306930
if (lookahead == '0') ADVANCE(142);
69316931
if (lookahead == ':') ADVANCE(70);
69326932
if (lookahead == '[') ADVANCE(66);
69336933
if (lookahead == ']') ADVANCE(67);
6934-
if (lookahead == '_') ADVANCE(83);
6934+
if (lookahead == '_') ADVANCE(82);
69356935
if (lookahead == 'b') ADVANCE(153);
69366936
if (lookahead == 'r') ADVANCE(155);
69376937
if (lookahead == '{') ADVANCE(63);
@@ -6943,7 +6943,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
69436943
if (('1' <= lookahead && lookahead <= '9')) ADVANCE(145);
69446944
if (('!' <= lookahead && lookahead <= '@') ||
69456945
lookahead == '^' ||
6946-
('|' <= lookahead && lookahead <= '~')) ADVANCE(84);
6946+
('|' <= lookahead && lookahead <= '~')) ADVANCE(83);
69476947
if (sym_identifier_character_set_3(lookahead)) ADVANCE(166);
69486948
END_STATE();
69496949
case 12:
@@ -6952,11 +6952,11 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
69526952
if (lookahead == '\'') ADVANCE(86);
69536953
if (lookahead == '(') ADVANCE(61);
69546954
if (lookahead == ')') ADVANCE(62);
6955-
if (lookahead == '/') ADVANCE(82);
6955+
if (lookahead == '/') ADVANCE(81);
69566956
if (lookahead == '0') ADVANCE(142);
69576957
if (lookahead == '[') ADVANCE(66);
69586958
if (lookahead == ']') ADVANCE(67);
6959-
if (lookahead == '_') ADVANCE(83);
6959+
if (lookahead == '_') ADVANCE(82);
69606960
if (lookahead == 'b') ADVANCE(153);
69616961
if (lookahead == 'r') ADVANCE(155);
69626962
if (lookahead == '{') ADVANCE(63);
@@ -6968,7 +6968,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
69686968
if (('1' <= lookahead && lookahead <= '9')) ADVANCE(145);
69696969
if (('!' <= lookahead && lookahead <= '@') ||
69706970
lookahead == '^' ||
6971-
('|' <= lookahead && lookahead <= '~')) ADVANCE(84);
6971+
('|' <= lookahead && lookahead <= '~')) ADVANCE(83);
69726972
if (sym_identifier_character_set_3(lookahead)) ADVANCE(166);
69736973
END_STATE();
69746974
case 13:
@@ -7366,7 +7366,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
73667366
lookahead == '^' ||
73677367
lookahead == '_' ||
73687368
lookahead == '|' ||
7369-
lookahead == '~') ADVANCE(84);
7369+
lookahead == '~') ADVANCE(83);
73707370
END_STATE();
73717371
case 71:
73727372
ACCEPT_TOKEN(anon_sym_DOLLAR);
@@ -7438,27 +7438,11 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
74387438
lookahead == '^' ||
74397439
lookahead == '_' ||
74407440
lookahead == '|' ||
7441-
lookahead == '~') ADVANCE(81);
7442-
if (lookahead != 0 &&
7443-
lookahead != '\n') ADVANCE(151);
7441+
lookahead == '~') ADVANCE(83);
74447442
END_STATE();
74457443
case 82:
74467444
ACCEPT_TOKEN(aux_sym__non_special_token_token1);
7447-
if (lookahead == '/') ADVANCE(81);
7448-
if (lookahead == '!' ||
7449-
lookahead == '#' ||
7450-
lookahead == '%' ||
7451-
lookahead == '&' ||
7452-
('*' <= lookahead && lookahead <= '.') ||
7453-
(':' <= lookahead && lookahead <= '@') ||
7454-
lookahead == '^' ||
7455-
lookahead == '_' ||
7456-
lookahead == '|' ||
7457-
lookahead == '~') ADVANCE(84);
7458-
END_STATE();
7459-
case 83:
7460-
ACCEPT_TOKEN(aux_sym__non_special_token_token1);
7461-
if (lookahead == '_') ADVANCE(83);
7445+
if (lookahead == '_') ADVANCE(82);
74627446
if (lookahead == '!' ||
74637447
lookahead == '#' ||
74647448
lookahead == '%' ||
@@ -7467,7 +7451,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
74677451
(':' <= lookahead && lookahead <= '@') ||
74687452
lookahead == '^' ||
74697453
lookahead == '|' ||
7470-
lookahead == '~') ADVANCE(84);
7454+
lookahead == '~') ADVANCE(83);
74717455
if (('0' <= lookahead && lookahead <= 'Z') ||
74727456
('a' <= lookahead && lookahead <= 'z') ||
74737457
lookahead == 170 ||
@@ -8266,6 +8250,19 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
82668250
(196608 <= lookahead && lookahead <= 201546) ||
82678251
(917760 <= lookahead && lookahead <= 917999)) ADVANCE(166);
82688252
END_STATE();
8253+
case 83:
8254+
ACCEPT_TOKEN(aux_sym__non_special_token_token1);
8255+
if (lookahead == '!' ||
8256+
lookahead == '#' ||
8257+
lookahead == '%' ||
8258+
lookahead == '&' ||
8259+
('*' <= lookahead && lookahead <= '/') ||
8260+
(':' <= lookahead && lookahead <= '@') ||
8261+
lookahead == '^' ||
8262+
lookahead == '_' ||
8263+
lookahead == '|' ||
8264+
lookahead == '~') ADVANCE(83);
8265+
END_STATE();
82698266
case 84:
82708267
ACCEPT_TOKEN(aux_sym__non_special_token_token1);
82718268
if (lookahead == '!' ||
@@ -8278,6 +8275,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
82788275
lookahead == '_' ||
82798276
lookahead == '|' ||
82808277
lookahead == '~') ADVANCE(84);
8278+
if (lookahead != 0 &&
8279+
lookahead != '\n') ADVANCE(151);
82818280
END_STATE();
82828281
case 85:
82838282
ACCEPT_TOKEN(anon_sym_SQUOTE);

0 commit comments

Comments
 (0)