Skip to content

Commit a756c9a

Browse files
VeykrilVeykril
committed
Fixup comments
1 parent 27cd509 commit a756c9a

File tree

7 files changed

+38
-26
lines changed

7 files changed

+38
-26
lines changed

‎crates/mbe/src/syntax_bridge.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ pub fn token_tree_to_syntax_node(
9595
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
9696
tree_sink.token(kind, n_raw_tokens)
9797
}
98-
parser::Step::FloatSplit { has_pseudo_dot } => tree_sink.float_split(has_pseudo_dot),
98+
parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
99+
tree_sink.float_split(has_pseudo_dot)
100+
}
99101
parser::Step::Enter { kind } => tree_sink.start_node(kind),
100102
parser::Step::Exit => tree_sink.finish_node(),
101103
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
@@ -797,6 +799,8 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
797799
}
798800

799801
impl<'a> TtTreeSink<'a> {
802+
/// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween.
803+
/// This occurs when a float literal is used as a field access.
800804
fn float_split(&mut self, has_pseudo_dot: bool) {
801805
let (text, _span) = match self.cursor.token_tree() {
802806
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => {

‎crates/mbe/src/to_parser_input.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
4747
res.push(kind);
4848

4949
if kind == FLOAT_NUMBER && !inner_text.ends_with('.') {
50+
// Tag the token as joint if it is float with a fractional part
51+
// we use this jointness to inform the parser about what token split
52+
// event to emit when we encounter a float literal in a field access
5053
res.was_joint();
5154
}
5255
}

‎crates/parser/src/event.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,14 @@ pub(crate) enum Event {
7272
/// `n_raw_tokens = 2` is used to produced a single `>>`.
7373
Token {
7474
kind: SyntaxKind,
75-
// Consider custom enum here?
7675
n_raw_tokens: u8,
7776
},
77+
/// When we parse `foo.0.0` or `foo. 0. 0` the lexer will hand us a float literal
78+
/// instead of an integer literal followed by a dot as the lexer has no contextual knowledge.
79+
/// This event instructs whatever consumes the events to split the float literal into
80+
/// the corresponding parts.
7881
FloatSplitHack {
79-
has_pseudo_dot: bool,
82+
ends_in_dot: bool,
8083
},
8184
Error {
8285
msg: String,
@@ -128,8 +131,8 @@ pub(super) fn process(mut events: Vec<Event>) -> Output {
128131
Event::Token { kind, n_raw_tokens } => {
129132
res.token(kind, n_raw_tokens);
130133
}
131-
Event::FloatSplitHack { has_pseudo_dot } => {
132-
res.float_split_hack(has_pseudo_dot);
134+
Event::FloatSplitHack { ends_in_dot } => {
135+
res.float_split_hack(ends_in_dot);
133136
let ev = mem::replace(&mut events[i + 1], Event::tombstone());
134137
assert!(matches!(ev, Event::Finish), "{ev:?}");
135138
}

‎crates/parser/src/lib.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,9 @@ impl TopEntryPoint {
102102
match step {
103103
Step::Enter { .. } => depth += 1,
104104
Step::Exit => depth -= 1,
105-
Step::FloatSplit { has_pseudo_dot } => depth -= 1 + !has_pseudo_dot as usize,
105+
Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
106+
depth -= 1 + !has_pseudo_dot as usize
107+
}
106108
Step::Token { .. } | Step::Error { .. } => (),
107109
}
108110
}

‎crates/parser/src/output.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ pub struct Output {
2525
#[derive(Debug)]
2626
pub enum Step<'a> {
2727
Token { kind: SyntaxKind, n_input_tokens: u8 },
28-
FloatSplit { has_pseudo_dot: bool },
28+
FloatSplit { ends_in_dot: bool },
2929
Enter { kind: SyntaxKind },
3030
Exit,
3131
Error { msg: &'a str },
@@ -70,7 +70,7 @@ impl Output {
7070
}
7171
Self::EXIT_EVENT => Step::Exit,
7272
Self::SPLIT_EVENT => {
73-
Step::FloatSplit { has_pseudo_dot: event & Self::N_INPUT_TOKEN_MASK != 0 }
73+
Step::FloatSplit { ends_in_dot: event & Self::N_INPUT_TOKEN_MASK != 0 }
7474
}
7575
_ => unreachable!(),
7676
}
@@ -84,9 +84,9 @@ impl Output {
8484
self.event.push(e)
8585
}
8686

87-
pub(crate) fn float_split_hack(&mut self, has_pseudo_dot: bool) {
87+
pub(crate) fn float_split_hack(&mut self, ends_in_dot: bool) {
8888
let e = (Self::SPLIT_EVENT as u32) << Self::TAG_SHIFT
89-
| ((has_pseudo_dot as u32) << Self::N_INPUT_TOKEN_SHIFT)
89+
| ((ends_in_dot as u32) << Self::N_INPUT_TOKEN_SHIFT)
9090
| Self::EVENT_MASK;
9191
self.event.push(e);
9292
}

‎crates/parser/src/parser.rs‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ impl<'t> Parser<'t> {
182182
}
183183

184184
/// Advances the parser by one token
185-
pub(crate) fn split_float(&mut self, marker: Marker) -> (bool, Marker) {
185+
pub(crate) fn split_float(&mut self, mut marker: Marker) -> (bool, Marker) {
186186
assert!(self.at(SyntaxKind::FLOAT_NUMBER));
187187
// we have parse `<something>.`
188188
// `<something>`.0.1
@@ -191,26 +191,23 @@ impl<'t> Parser<'t> {
191191
// `<something>`. 0. 1;
192192
// here we need to change the follow up parse, the return value will cause us to emulate a dot
193193
// the actual splitting happens later
194-
let has_pseudo_dot = !self.inp.is_joint(self.pos);
195-
let marker = if !has_pseudo_dot {
196-
let new_pos = self.start();
194+
let ends_in_dot = !self.inp.is_joint(self.pos);
195+
if !ends_in_dot {
196+
let new_marker = self.start();
197197
let idx = marker.pos as usize;
198198
match &mut self.events[idx] {
199199
Event::Start { forward_parent, kind } => {
200200
*kind = SyntaxKind::FIELD_EXPR;
201-
*forward_parent = Some(new_pos.pos - marker.pos);
201+
*forward_parent = Some(new_marker.pos - marker.pos);
202202
}
203203
_ => unreachable!(),
204204
}
205-
// NOTE: This brings the start / finish pairs out of balance!
206-
std::mem::forget(marker);
207-
new_pos
208-
} else {
209-
marker
205+
marker.bomb.defuse();
206+
marker = new_marker;
210207
};
211208
self.pos += 1 as usize;
212-
self.push_event(Event::FloatSplitHack { has_pseudo_dot });
213-
(has_pseudo_dot, marker)
209+
self.push_event(Event::FloatSplitHack { ends_in_dot });
210+
(ends_in_dot, marker)
214211
}
215212

216213
/// Advances the parser by one token, remapping its kind.

‎crates/parser/src/shortcuts.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ impl<'a> LexedStr<'a> {
4343
res.was_joint();
4444
}
4545
res.push(kind);
46-
// we set jointness for floating point numbers as a hack to inform the
47-
// parser about whether we have a `0.` or `0.1` style float
46+
// Tag the token as joint if it is float with a fractional part
47+
// we use this jointness to inform the parser about what token split
48+
// event to emit when we encounter a float literal in a field access
4849
if kind == SyntaxKind::FLOAT_NUMBER {
49-
if !self.text(i).split_once('.').map_or(true, |(_, it)| it.is_empty()) {
50+
if !self.text(i).ends_with('.') {
5051
res.was_joint();
5152
}
5253
}
@@ -71,7 +72,9 @@ impl<'a> LexedStr<'a> {
7172
Step::Token { kind, n_input_tokens: n_raw_tokens } => {
7273
builder.token(kind, n_raw_tokens)
7374
}
74-
Step::FloatSplit { has_pseudo_dot } => builder.float_split(has_pseudo_dot),
75+
Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
76+
builder.float_split(has_pseudo_dot)
77+
}
7578
Step::Enter { kind } => builder.enter(kind),
7679
Step::Exit => builder.exit(),
7780
Step::Error { msg } => {

0 commit comments

Comments
 (0)