Skip to content

Commit 0182fd9

Browse files
borsbors
committed
Auto merge of rust-lang#98186 - mystor:tokenstream_as_vec_tt, r=eddyb
Batch proc_macro RPC for TokenStream iteration and combination operations This is the first part of rust-lang#86822, split off as requested in rust-lang#86822 (review). It reduces the number of RPC calls required for common operations such as iterating over and concatenating TokenStreams.
2 parents ff86b27 + df925fd commit 0182fd9

File tree

6 files changed

+303
-173
lines changed

6 files changed

+303
-173
lines changed

‎compiler/rustc_expand/src/proc_macro_server.rs‎

Lines changed: 55 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -277,12 +277,6 @@ impl ToInternal<rustc_errors::Level> for Level {
277277

278278
pub struct FreeFunctions;
279279

280-
#[derive(Clone)]
281-
pub struct TokenStreamIter {
282-
cursor: tokenstream::Cursor,
283-
stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
284-
}
285-
286280
#[derive(Clone)]
287281
pub struct Group {
288282
delimiter: Delimiter,
@@ -382,8 +376,6 @@ impl<'a, 'b> Rustc<'a, 'b> {
382376
impl server::Types for Rustc<'_, '_> {
383377
type FreeFunctions = FreeFunctions;
384378
type TokenStream = TokenStream;
385-
type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
386-
type TokenStreamIter = TokenStreamIter;
387379
type Group = Group;
388380
type Punct = Punct;
389381
type Ident = Ident;
@@ -408,9 +400,6 @@ impl server::FreeFunctions for Rustc<'_, '_> {
408400
}
409401

410402
impl server::TokenStream for Rustc<'_, '_> {
411-
fn new(&mut self) -> Self::TokenStream {
412-
TokenStream::default()
413-
}
414403
fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
415404
stream.is_empty()
416405
}
@@ -481,53 +470,75 @@ impl server::TokenStream for Rustc<'_, '_> {
481470
) -> Self::TokenStream {
482471
tree.to_internal()
483472
}
484-
fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
485-
TokenStreamIter { cursor: stream.into_trees(), stack: vec![] }
486-
}
487-
}
488-
489-
impl server::TokenStreamBuilder for Rustc<'_, '_> {
490-
fn new(&mut self) -> Self::TokenStreamBuilder {
491-
tokenstream::TokenStreamBuilder::new()
492-
}
493-
fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
494-
builder.push(stream);
473+
fn concat_trees(
474+
&mut self,
475+
base: Option<Self::TokenStream>,
476+
trees: Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>>,
477+
) -> Self::TokenStream {
478+
let mut builder = tokenstream::TokenStreamBuilder::new();
479+
if let Some(base) = base {
480+
builder.push(base);
481+
}
482+
for tree in trees {
483+
builder.push(tree.to_internal());
484+
}
485+
builder.build()
495486
}
496-
fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
487+
fn concat_streams(
488+
&mut self,
489+
base: Option<Self::TokenStream>,
490+
streams: Vec<Self::TokenStream>,
491+
) -> Self::TokenStream {
492+
let mut builder = tokenstream::TokenStreamBuilder::new();
493+
if let Some(base) = base {
494+
builder.push(base);
495+
}
496+
for stream in streams {
497+
builder.push(stream);
498+
}
497499
builder.build()
498500
}
499-
}
500-
501-
impl server::TokenStreamIter for Rustc<'_, '_> {
502-
fn next(
501+
fn into_trees(
503502
&mut self,
504-
iter: &mut Self::TokenStreamIter,
505-
) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
503+
stream: Self::TokenStream,
504+
) -> Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
505+
// FIXME: This is a raw port of the previous approach (which had a
506+
// `TokenStreamIter` server-side object with a single `next` method),
507+
// and can probably be optimized (for bulk conversion).
508+
let mut cursor = stream.into_trees();
509+
let mut stack = Vec::new();
510+
let mut tts = Vec::new();
506511
loop {
507-
let tree = iter.stack.pop().or_else(|| {
508-
let next = iter.cursor.next_with_spacing()?;
509-
Some(TokenTree::from_internal((next, &mut iter.stack, self)))
510-
})?;
511-
// A hack used to pass AST fragments to attribute and derive macros
512-
// as a single nonterminal token instead of a token stream.
513-
// Such token needs to be "unwrapped" and not represented as a delimited group.
514-
// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
515-
if let TokenTree::Group(ref group) = tree {
516-
if group.flatten {
517-
iter.cursor.append(group.stream.clone());
518-
continue;
512+
let next = stack.pop().or_else(|| {
513+
let next = cursor.next_with_spacing()?;
514+
Some(TokenTree::from_internal((next, &mut stack, self)))
515+
});
516+
match next {
517+
Some(TokenTree::Group(group)) => {
518+
// A hack used to pass AST fragments to attribute and derive
519+
// macros as a single nonterminal token instead of a token
520+
// stream. Such token needs to be "unwrapped" and not
521+
// represented as a delimited group.
522+
// FIXME: It needs to be removed, but there are some
523+
// compatibility issues (see #73345).
524+
if group.flatten {
525+
cursor.append(group.stream);
526+
continue;
527+
}
528+
tts.push(TokenTree::Group(group));
519529
}
530+
Some(tt) => tts.push(tt),
531+
None => return tts,
520532
}
521-
return Some(tree);
522533
}
523534
}
524535
}
525536

526537
impl server::Group for Rustc<'_, '_> {
527-
fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
538+
fn new(&mut self, delimiter: Delimiter, stream: Option<Self::TokenStream>) -> Self::Group {
528539
Group {
529540
delimiter,
530-
stream,
541+
stream: stream.unwrap_or_default(),
531542
span: DelimSpan::from_single(server::Span::call_site(self)),
532543
flatten: false,
533544
}

‎library/proc_macro/src/bridge/client.rs‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,6 @@ define_handles! {
178178
'owned:
179179
FreeFunctions,
180180
TokenStream,
181-
TokenStreamBuilder,
182-
TokenStreamIter,
183181
Group,
184182
Literal,
185183
SourceFile,
@@ -204,12 +202,6 @@ impl Clone for TokenStream {
204202
}
205203
}
206204

207-
impl Clone for TokenStreamIter {
208-
fn clone(&self) -> Self {
209-
self.clone()
210-
}
211-
}
212-
213205
impl Clone for Group {
214206
fn clone(&self) -> Self {
215207
self.clone()
@@ -435,7 +427,7 @@ impl Client<crate::TokenStream, crate::TokenStream> {
435427
Client {
436428
get_handle_counters: HandleCounters::get,
437429
run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| {
438-
run_client(bridge, |input| f(crate::TokenStream(input)).0)
430+
run_client(bridge, |input| f(crate::TokenStream(Some(input))).0)
439431
}),
440432
_marker: PhantomData,
441433
}
@@ -450,7 +442,7 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> {
450442
get_handle_counters: HandleCounters::get,
451443
run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| {
452444
run_client(bridge, |(input, input2)| {
453-
f(crate::TokenStream(input), crate::TokenStream(input2)).0
445+
f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))).0
454446
})
455447
}),
456448
_marker: PhantomData,

‎library/proc_macro/src/bridge/mod.rs‎

Lines changed: 76 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -60,33 +60,29 @@ macro_rules! with_api {
6060
TokenStream {
6161
fn drop($self: $S::TokenStream);
6262
fn clone($self: &$S::TokenStream) -> $S::TokenStream;
63-
fn new() -> $S::TokenStream;
6463
fn is_empty($self: &$S::TokenStream) -> bool;
6564
fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
6665
fn from_str(src: &str) -> $S::TokenStream;
6766
fn to_string($self: &$S::TokenStream) -> String;
6867
fn from_token_tree(
6968
tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>,
7069
) -> $S::TokenStream;
71-
fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter;
72-
},
73-
TokenStreamBuilder {
74-
fn drop($self: $S::TokenStreamBuilder);
75-
fn new() -> $S::TokenStreamBuilder;
76-
fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream);
77-
fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream;
78-
},
79-
TokenStreamIter {
80-
fn drop($self: $S::TokenStreamIter);
81-
fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter;
82-
fn next(
83-
$self: &mut $S::TokenStreamIter,
84-
) -> Option<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
70+
fn concat_trees(
71+
base: Option<$S::TokenStream>,
72+
trees: Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>,
73+
) -> $S::TokenStream;
74+
fn concat_streams(
75+
base: Option<$S::TokenStream>,
76+
streams: Vec<$S::TokenStream>,
77+
) -> $S::TokenStream;
78+
fn into_trees(
79+
$self: $S::TokenStream
80+
) -> Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
8581
},
8682
Group {
8783
fn drop($self: $S::Group);
8884
fn clone($self: &$S::Group) -> $S::Group;
89-
fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group;
85+
fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group;
9086
fn delimiter($self: &$S::Group) -> Delimiter;
9187
fn stream($self: &$S::Group) -> $S::TokenStream;
9288
fn span($self: &$S::Group) -> $S::Span;
@@ -311,29 +307,18 @@ impl<'a, T, M> Unmark for &'a mut Marked<T, M> {
311307
}
312308
}
313309

314-
impl<T: Mark> Mark for Option<T> {
315-
type Unmarked = Option<T::Unmarked>;
316-
fn mark(unmarked: Self::Unmarked) -> Self {
317-
unmarked.map(T::mark)
318-
}
319-
}
320-
impl<T: Unmark> Unmark for Option<T> {
321-
type Unmarked = Option<T::Unmarked>;
322-
fn unmark(self) -> Self::Unmarked {
323-
self.map(T::unmark)
324-
}
325-
}
326-
327-
impl<T: Mark, E: Mark> Mark for Result<T, E> {
328-
type Unmarked = Result<T::Unmarked, E::Unmarked>;
310+
impl<T: Mark> Mark for Vec<T> {
311+
type Unmarked = Vec<T::Unmarked>;
329312
fn mark(unmarked: Self::Unmarked) -> Self {
330-
unmarked.map(T::mark).map_err(E::mark)
313+
// Should be a no-op due to std's in-place collect optimizations.
314+
unmarked.into_iter().map(T::mark).collect()
331315
}
332316
}
333-
impl<T: Unmark, E: Unmark> Unmark for Result<T, E> {
334-
type Unmarked = Result<T::Unmarked, E::Unmarked>;
317+
impl<T: Unmark> Unmark for Vec<T> {
318+
type Unmarked = Vec<T::Unmarked>;
335319
fn unmark(self) -> Self::Unmarked {
336-
self.map(T::unmark).map_err(E::unmark)
320+
// Should be a no-op due to std's in-place collect optimizations.
321+
self.into_iter().map(T::unmark).collect()
337322
}
338323
}
339324

@@ -367,7 +352,6 @@ mark_noop! {
367352
Level,
368353
LineColumn,
369354
Spacing,
370-
Bound<usize>,
371355
}
372356

373357
rpc_encode_decode!(
@@ -394,6 +378,61 @@ rpc_encode_decode!(
394378
}
395379
);
396380

381+
macro_rules! mark_compound {
382+
(enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => {
383+
impl<$($T: Mark),+> Mark for $name <$($T),+> {
384+
type Unmarked = $name <$($T::Unmarked),+>;
385+
fn mark(unmarked: Self::Unmarked) -> Self {
386+
match unmarked {
387+
$($name::$variant $(($field))? => {
388+
$name::$variant $((Mark::mark($field)))?
389+
})*
390+
}
391+
}
392+
}
393+
394+
impl<$($T: Unmark),+> Unmark for $name <$($T),+> {
395+
type Unmarked = $name <$($T::Unmarked),+>;
396+
fn unmark(self) -> Self::Unmarked {
397+
match self {
398+
$($name::$variant $(($field))? => {
399+
$name::$variant $((Unmark::unmark($field)))?
400+
})*
401+
}
402+
}
403+
}
404+
}
405+
}
406+
407+
macro_rules! compound_traits {
408+
($($t:tt)*) => {
409+
rpc_encode_decode!($($t)*);
410+
mark_compound!($($t)*);
411+
};
412+
}
413+
414+
compound_traits!(
415+
enum Bound<T> {
416+
Included(x),
417+
Excluded(x),
418+
Unbounded,
419+
}
420+
);
421+
422+
compound_traits!(
423+
enum Option<T> {
424+
Some(t),
425+
None,
426+
}
427+
);
428+
429+
compound_traits!(
430+
enum Result<T, E> {
431+
Ok(t),
432+
Err(e),
433+
}
434+
);
435+
397436
#[derive(Clone)]
398437
pub enum TokenTree<G, P, I, L> {
399438
Group(G),
@@ -402,30 +441,7 @@ pub enum TokenTree<G, P, I, L> {
402441
Literal(L),
403442
}
404443

405-
impl<G: Mark, P: Mark, I: Mark, L: Mark> Mark for TokenTree<G, P, I, L> {
406-
type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
407-
fn mark(unmarked: Self::Unmarked) -> Self {
408-
match unmarked {
409-
TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)),
410-
TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)),
411-
TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)),
412-
TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)),
413-
}
414-
}
415-
}
416-
impl<G: Unmark, P: Unmark, I: Unmark, L: Unmark> Unmark for TokenTree<G, P, I, L> {
417-
type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
418-
fn unmark(self) -> Self::Unmarked {
419-
match self {
420-
TokenTree::Group(tt) => TokenTree::Group(tt.unmark()),
421-
TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()),
422-
TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()),
423-
TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()),
424-
}
425-
}
426-
}
427-
428-
rpc_encode_decode!(
444+
compound_traits!(
429445
enum TokenTree<G, P, I, L> {
430446
Group(tt),
431447
Punct(tt),

0 commit comments

Comments
 (0)