Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 890e759

Browse files
nnethercotennethercote
committed
Move Spacing out of AttrAnnotatedTokenStream.
And into `AttrAnnotatedTokenTree::Token`. PR rust-lang#99887 did the same thing for `TokenStream`.
1 parent 1120c5e commit 890e759

File tree

5 files changed

+34
-44
lines changed

5 files changed

+34
-44
lines changed

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -303,13 +303,12 @@ impl Attribute {
303303
.as_ref()
304304
.unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
305305
.create_token_stream(),
306-
AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
307-
AttrAnnotatedTokenTree::Token(Token::new(
308-
token::DocComment(comment_kind, self.style, data),
309-
self.span,
310-
)),
311-
Spacing::Alone,
312-
)),
306+
AttrKind::DocComment(comment_kind, data) => {
307+
AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Token(
308+
Token::new(token::DocComment(comment_kind, self.style, data), self.span),
309+
Spacing::Alone,
310+
)])
311+
}
313312
}
314313
}
315314
}

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
644644
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
645645
pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
646646
match tt {
647-
AttrAnnotatedTokenTree::Token(token) => {
647+
AttrAnnotatedTokenTree::Token(token, _) => {
648648
visit_token(token, vis);
649649
}
650650
AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
@@ -696,7 +696,7 @@ pub fn visit_attr_annotated_tts<T: MutVisitor>(
696696
) {
697697
if T::VISIT_TOKENS && !tts.is_empty() {
698698
let tts = Lrc::make_mut(tts);
699-
visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
699+
visit_vec(tts, |tree| visit_attr_annotated_tt(tree, vis));
700700
}
701701
}
702702

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -177,12 +177,12 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
177177
/// during expansion to perform early cfg-expansion, and to process attributes
178178
/// during proc-macro invocations.
179179
#[derive(Clone, Debug, Default, Encodable, Decodable)]
180-
pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>);
180+
pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<AttrAnnotatedTokenTree>>);
181181

182182
/// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
183183
#[derive(Clone, Debug, Encodable, Decodable)]
184184
pub enum AttrAnnotatedTokenTree {
185-
Token(Token),
185+
Token(Token, Spacing),
186186
Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream),
187187
/// Stores the attributes for an attribute target,
188188
/// along with the tokens for that attribute target.
@@ -191,7 +191,7 @@ pub enum AttrAnnotatedTokenTree {
191191
}
192192

193193
impl AttrAnnotatedTokenStream {
194-
pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream {
194+
pub fn new(tokens: Vec<AttrAnnotatedTokenTree>) -> AttrAnnotatedTokenStream {
195195
AttrAnnotatedTokenStream(Lrc::new(tokens))
196196
}
197197

@@ -204,9 +204,9 @@ impl AttrAnnotatedTokenStream {
204204
let trees: Vec<_> = self
205205
.0
206206
.iter()
207-
.flat_map(|tree| match &tree.0 {
208-
AttrAnnotatedTokenTree::Token(inner) => {
209-
smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter()
207+
.flat_map(|tree| match &tree {
208+
AttrAnnotatedTokenTree::Token(inner, spacing) => {
209+
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
210210
}
211211
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
212212
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
@@ -363,12 +363,6 @@ impl TokenStream {
363363
}
364364
}
365365

366-
impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
367-
fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
368-
AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
369-
}
370-
}
371-
372366
impl iter::FromIterator<TokenTree> for TokenStream {
373367
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
374368
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
@@ -428,10 +422,7 @@ impl TokenStream {
428422
} else {
429423
let attr_data =
430424
AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
431-
AttrAnnotatedTokenStream::new(vec![(
432-
AttrAnnotatedTokenTree::Attributes(attr_data),
433-
Spacing::Alone,
434-
)])
425+
AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Attributes(attr_data)])
435426
};
436427
Some(attr_annotated.to_tokenstream())
437428
}

‎compiler/rustc_expand/src/config.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -276,9 +276,9 @@ impl<'a> StripUnconfigured<'a> {
276276
/// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
277277
fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
278278
fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
279-
stream.0.iter().all(|(tree, _spacing)| match tree {
279+
stream.0.iter().all(|tree| match tree {
280280
AttrAnnotatedTokenTree::Attributes(_) => false,
281-
AttrAnnotatedTokenTree::Token(_) => true,
281+
AttrAnnotatedTokenTree::Token(..) => true,
282282
AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
283283
})
284284
}
@@ -290,32 +290,32 @@ impl<'a> StripUnconfigured<'a> {
290290
let trees: Vec<_> = stream
291291
.0
292292
.iter()
293-
.flat_map(|(tree, spacing)| match tree.clone() {
293+
.flat_map(|tree| match tree.clone() {
294294
AttrAnnotatedTokenTree::Attributes(mut data) => {
295295
data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
296296

297297
if self.in_cfg(&data.attrs) {
298298
data.tokens = LazyTokenStream::new(
299299
self.configure_tokens(&data.tokens.create_token_stream()),
300300
);
301-
Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter()
301+
Some(AttrAnnotatedTokenTree::Attributes(data)).into_iter()
302302
} else {
303303
None.into_iter()
304304
}
305305
}
306306
AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
307307
inner = self.configure_tokens(&inner);
308-
Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing))
308+
Some(AttrAnnotatedTokenTree::Delimited(sp, delim, inner))
309309
.into_iter()
310310
}
311-
AttrAnnotatedTokenTree::Token(ref token) if let TokenKind::Interpolated(ref nt) = token.kind => {
311+
AttrAnnotatedTokenTree::Token(ref token, _) if let TokenKind::Interpolated(ref nt) = token.kind => {
312312
panic!(
313313
"Nonterminal should have been flattened at {:?}: {:?}",
314314
token.span, nt
315315
);
316316
}
317-
AttrAnnotatedTokenTree::Token(token) => {
318-
Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter()
317+
AttrAnnotatedTokenTree::Token(token, spacing) => {
318+
Some(AttrAnnotatedTokenTree::Token(token, spacing)).into_iter()
319319
}
320320
})
321321
.collect();
@@ -404,13 +404,13 @@ impl<'a> StripUnconfigured<'a> {
404404
};
405405
let pound_span = pound_token.span;
406406

407-
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
407+
let mut trees = vec![AttrAnnotatedTokenTree::Token(pound_token, Spacing::Alone)];
408408
if attr.style == AttrStyle::Inner {
409409
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
410410
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
411411
panic!("Bad tokens for attribute {:?}", attr);
412412
};
413-
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
413+
trees.push(AttrAnnotatedTokenTree::Token(bang_token, Spacing::Alone));
414414
}
415415
// We don't really have a good span to use for the synthesized `[]`
416416
// in `#[attr]`, so just use the span of the `#` token.
@@ -422,7 +422,7 @@ impl<'a> StripUnconfigured<'a> {
422422
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
423423
.create_token_stream(),
424424
);
425-
trees.push((bracket_group, Spacing::Alone));
425+
trees.push(bracket_group);
426426
let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
427427
let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span);
428428
if attr.has_name(sym::crate_type) {

‎compiler/rustc_parse/src/parser/attr_wrapper.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,7 @@ fn make_token_stream(
397397
struct FrameData {
398398
// This is `None` for the first frame, `Some` for all others.
399399
open_delim_sp: Option<(Delimiter, Span)>,
400-
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
400+
inner: Vec<AttrAnnotatedTokenTree>,
401401
}
402402
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
403403
let mut token_and_spacing = iter.next();
@@ -426,34 +426,34 @@ fn make_token_stream(
426426
panic!("Bottom token frame is missing for token: {:?}", token)
427427
})
428428
.inner
429-
.push((delimited, Spacing::Alone));
429+
.push(delimited);
430430
}
431431
FlatToken::Token(token) => stack
432432
.last_mut()
433433
.expect("Bottom token frame is missing!")
434434
.inner
435-
.push((AttrAnnotatedTokenTree::Token(token), spacing)),
435+
.push(AttrAnnotatedTokenTree::Token(token, spacing)),
436436
FlatToken::AttrTarget(data) => stack
437437
.last_mut()
438438
.expect("Bottom token frame is missing!")
439439
.inner
440-
.push((AttrAnnotatedTokenTree::Attributes(data), spacing)),
440+
.push(AttrAnnotatedTokenTree::Attributes(data)),
441441
FlatToken::Empty => {}
442442
}
443443
token_and_spacing = iter.next();
444444
}
445445
let mut final_buf = stack.pop().expect("Missing final buf!");
446446
if break_last_token {
447-
let (last_token, spacing) = final_buf.inner.pop().unwrap();
448-
if let AttrAnnotatedTokenTree::Token(last_token) = last_token {
447+
let last_token = final_buf.inner.pop().unwrap();
448+
if let AttrAnnotatedTokenTree::Token(last_token, spacing) = last_token {
449449
let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
450450

451451
// An 'unglued' token is always two ASCII characters
452452
let mut first_span = last_token.span.shrink_to_lo();
453453
first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
454454

455-
final_buf.inner.push((
456-
AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)),
455+
final_buf.inner.push(AttrAnnotatedTokenTree::Token(
456+
Token::new(unglued_first, first_span),
457457
spacing,
458458
));
459459
} else {

0 commit comments

Comments
 (0)