Skip to content

Commit 1d5f643

Browse files
borsbors
committed
Auto merge of #98186 - mystor:tokenstream_as_vec_tt, r=eddyb
Batch proc_macro RPC for TokenStream iteration and combination operations This is the first part of #86822, split off as requested in rust-lang/rust#86822 (review). It reduces the number of RPC calls required for common operations such as iterating over and concatenating TokenStreams.
2 parents 81b0162 + 4075903 commit 1d5f643

File tree

5 files changed

+248
-129
lines changed

5 files changed

+248
-129
lines changed

‎proc_macro/src/bridge/client.rs‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,6 @@ define_handles! {
178178
'owned:
179179
FreeFunctions,
180180
TokenStream,
181-
TokenStreamBuilder,
182-
TokenStreamIter,
183181
Group,
184182
Literal,
185183
SourceFile,
@@ -204,12 +202,6 @@ impl Clone for TokenStream {
204202
}
205203
}
206204

207-
impl Clone for TokenStreamIter {
208-
fn clone(&self) -> Self {
209-
self.clone()
210-
}
211-
}
212-
213205
impl Clone for Group {
214206
fn clone(&self) -> Self {
215207
self.clone()
@@ -435,7 +427,7 @@ impl Client<crate::TokenStream, crate::TokenStream> {
435427
Client {
436428
get_handle_counters: HandleCounters::get,
437429
run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| {
438-
run_client(bridge, |input| f(crate::TokenStream(input)).0)
430+
run_client(bridge, |input| f(crate::TokenStream(Some(input))).0)
439431
}),
440432
_marker: PhantomData,
441433
}
@@ -450,7 +442,7 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> {
450442
get_handle_counters: HandleCounters::get,
451443
run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| {
452444
run_client(bridge, |(input, input2)| {
453-
f(crate::TokenStream(input), crate::TokenStream(input2)).0
445+
f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))).0
454446
})
455447
}),
456448
_marker: PhantomData,

‎proc_macro/src/bridge/mod.rs‎

Lines changed: 76 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -60,33 +60,29 @@ macro_rules! with_api {
6060
TokenStream {
6161
fn drop($self: $S::TokenStream);
6262
fn clone($self: &$S::TokenStream) -> $S::TokenStream;
63-
fn new() -> $S::TokenStream;
6463
fn is_empty($self: &$S::TokenStream) -> bool;
6564
fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
6665
fn from_str(src: &str) -> $S::TokenStream;
6766
fn to_string($self: &$S::TokenStream) -> String;
6867
fn from_token_tree(
6968
tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>,
7069
) -> $S::TokenStream;
71-
fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter;
72-
},
73-
TokenStreamBuilder {
74-
fn drop($self: $S::TokenStreamBuilder);
75-
fn new() -> $S::TokenStreamBuilder;
76-
fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream);
77-
fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream;
78-
},
79-
TokenStreamIter {
80-
fn drop($self: $S::TokenStreamIter);
81-
fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter;
82-
fn next(
83-
$self: &mut $S::TokenStreamIter,
84-
) -> Option<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
70+
fn concat_trees(
71+
base: Option<$S::TokenStream>,
72+
trees: Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>,
73+
) -> $S::TokenStream;
74+
fn concat_streams(
75+
base: Option<$S::TokenStream>,
76+
streams: Vec<$S::TokenStream>,
77+
) -> $S::TokenStream;
78+
fn into_trees(
79+
$self: $S::TokenStream
80+
) -> Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
8581
},
8682
Group {
8783
fn drop($self: $S::Group);
8884
fn clone($self: &$S::Group) -> $S::Group;
89-
fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group;
85+
fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group;
9086
fn delimiter($self: &$S::Group) -> Delimiter;
9187
fn stream($self: &$S::Group) -> $S::TokenStream;
9288
fn span($self: &$S::Group) -> $S::Span;
@@ -311,29 +307,18 @@ impl<'a, T, M> Unmark for &'a mut Marked<T, M> {
311307
}
312308
}
313309

314-
impl<T: Mark> Mark for Option<T> {
315-
type Unmarked = Option<T::Unmarked>;
316-
fn mark(unmarked: Self::Unmarked) -> Self {
317-
unmarked.map(T::mark)
318-
}
319-
}
320-
impl<T: Unmark> Unmark for Option<T> {
321-
type Unmarked = Option<T::Unmarked>;
322-
fn unmark(self) -> Self::Unmarked {
323-
self.map(T::unmark)
324-
}
325-
}
326-
327-
impl<T: Mark, E: Mark> Mark for Result<T, E> {
328-
type Unmarked = Result<T::Unmarked, E::Unmarked>;
310+
impl<T: Mark> Mark for Vec<T> {
311+
type Unmarked = Vec<T::Unmarked>;
329312
fn mark(unmarked: Self::Unmarked) -> Self {
330-
unmarked.map(T::mark).map_err(E::mark)
313+
// Should be a no-op due to std's in-place collect optimizations.
314+
unmarked.into_iter().map(T::mark).collect()
331315
}
332316
}
333-
impl<T: Unmark, E: Unmark> Unmark for Result<T, E> {
334-
type Unmarked = Result<T::Unmarked, E::Unmarked>;
317+
impl<T: Unmark> Unmark for Vec<T> {
318+
type Unmarked = Vec<T::Unmarked>;
335319
fn unmark(self) -> Self::Unmarked {
336-
self.map(T::unmark).map_err(E::unmark)
320+
// Should be a no-op due to std's in-place collect optimizations.
321+
self.into_iter().map(T::unmark).collect()
337322
}
338323
}
339324

@@ -367,7 +352,6 @@ mark_noop! {
367352
Level,
368353
LineColumn,
369354
Spacing,
370-
Bound<usize>,
371355
}
372356

373357
rpc_encode_decode!(
@@ -394,6 +378,61 @@ rpc_encode_decode!(
394378
}
395379
);
396380

381+
macro_rules! mark_compound {
382+
(enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => {
383+
impl<$($T: Mark),+> Mark for $name <$($T),+> {
384+
type Unmarked = $name <$($T::Unmarked),+>;
385+
fn mark(unmarked: Self::Unmarked) -> Self {
386+
match unmarked {
387+
$($name::$variant $(($field))? => {
388+
$name::$variant $((Mark::mark($field)))?
389+
})*
390+
}
391+
}
392+
}
393+
394+
impl<$($T: Unmark),+> Unmark for $name <$($T),+> {
395+
type Unmarked = $name <$($T::Unmarked),+>;
396+
fn unmark(self) -> Self::Unmarked {
397+
match self {
398+
$($name::$variant $(($field))? => {
399+
$name::$variant $((Unmark::unmark($field)))?
400+
})*
401+
}
402+
}
403+
}
404+
}
405+
}
406+
407+
macro_rules! compound_traits {
408+
($($t:tt)*) => {
409+
rpc_encode_decode!($($t)*);
410+
mark_compound!($($t)*);
411+
};
412+
}
413+
414+
compound_traits!(
415+
enum Bound<T> {
416+
Included(x),
417+
Excluded(x),
418+
Unbounded,
419+
}
420+
);
421+
422+
compound_traits!(
423+
enum Option<T> {
424+
Some(t),
425+
None,
426+
}
427+
);
428+
429+
compound_traits!(
430+
enum Result<T, E> {
431+
Ok(t),
432+
Err(e),
433+
}
434+
);
435+
397436
#[derive(Clone)]
398437
pub enum TokenTree<G, P, I, L> {
399438
Group(G),
@@ -402,30 +441,7 @@ pub enum TokenTree<G, P, I, L> {
402441
Literal(L),
403442
}
404443

405-
impl<G: Mark, P: Mark, I: Mark, L: Mark> Mark for TokenTree<G, P, I, L> {
406-
type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
407-
fn mark(unmarked: Self::Unmarked) -> Self {
408-
match unmarked {
409-
TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)),
410-
TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)),
411-
TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)),
412-
TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)),
413-
}
414-
}
415-
}
416-
impl<G: Unmark, P: Unmark, I: Unmark, L: Unmark> Unmark for TokenTree<G, P, I, L> {
417-
type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
418-
fn unmark(self) -> Self::Unmarked {
419-
match self {
420-
TokenTree::Group(tt) => TokenTree::Group(tt.unmark()),
421-
TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()),
422-
TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()),
423-
TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()),
424-
}
425-
}
426-
}
427-
428-
rpc_encode_decode!(
444+
compound_traits!(
429445
enum TokenTree<G, P, I, L> {
430446
Group(tt),
431447
Punct(tt),

‎proc_macro/src/bridge/rpc.rs‎

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ use std::any::Any;
44
use std::char;
55
use std::io::Write;
66
use std::num::NonZeroU32;
7-
use std::ops::Bound;
87
use std::str;
98

109
pub(super) type Writer = super::buffer::Buffer;
@@ -43,15 +42,17 @@ macro_rules! rpc_encode_decode {
4342
}
4443
}
4544
};
46-
(struct $name:ident { $($field:ident),* $(,)? }) => {
47-
impl<S> Encode<S> for $name {
45+
(struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => {
46+
impl<S, $($($T: Encode<S>),+)?> Encode<S> for $name $(<$($T),+>)? {
4847
fn encode(self, w: &mut Writer, s: &mut S) {
4948
$(self.$field.encode(w, s);)*
5049
}
5150
}
5251

53-
impl<S> DecodeMut<'_, '_, S> for $name {
54-
fn decode(r: &mut Reader<'_>, s: &mut S) -> Self {
52+
impl<'a, S, $($($T: for<'s> DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S>
53+
for $name $(<$($T),+>)?
54+
{
55+
fn decode(r: &mut Reader<'a>, s: &mut S) -> Self {
5556
$name {
5657
$($field: DecodeMut::decode(r, s)),*
5758
}
@@ -184,28 +185,6 @@ impl<'a, S, A: for<'s> DecodeMut<'a, 's, S>, B: for<'s> DecodeMut<'a, 's, S>> De
184185
}
185186
}
186187

187-
rpc_encode_decode!(
188-
enum Bound<T> {
189-
Included(x),
190-
Excluded(x),
191-
Unbounded,
192-
}
193-
);
194-
195-
rpc_encode_decode!(
196-
enum Option<T> {
197-
None,
198-
Some(x),
199-
}
200-
);
201-
202-
rpc_encode_decode!(
203-
enum Result<T, E> {
204-
Ok(x),
205-
Err(e),
206-
}
207-
);
208-
209188
impl<S> Encode<S> for &[u8] {
210189
fn encode(self, w: &mut Writer, s: &mut S) {
211190
self.len().encode(w, s);
@@ -246,6 +225,26 @@ impl<S> DecodeMut<'_, '_, S> for String {
246225
}
247226
}
248227

228+
impl<S, T: Encode<S>> Encode<S> for Vec<T> {
229+
fn encode(self, w: &mut Writer, s: &mut S) {
230+
self.len().encode(w, s);
231+
for x in self {
232+
x.encode(w, s);
233+
}
234+
}
235+
}
236+
237+
impl<'a, S, T: for<'s> DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec<T> {
238+
fn decode(r: &mut Reader<'a>, s: &mut S) -> Self {
239+
let len = usize::decode(r, s);
240+
let mut vec = Vec::with_capacity(len);
241+
for _ in 0..len {
242+
vec.push(T::decode(r, s));
243+
}
244+
vec
245+
}
246+
}
247+
249248
/// Simplified version of panic payloads, ignoring
250249
/// types other than `&'static str` and `String`.
251250
pub enum PanicMessage {

‎proc_macro/src/bridge/server.rs‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ use super::client::HandleStore;
88
pub trait Types {
99
type FreeFunctions: 'static;
1010
type TokenStream: 'static + Clone;
11-
type TokenStreamBuilder: 'static;
12-
type TokenStreamIter: 'static + Clone;
1311
type Group: 'static + Clone;
1412
type Punct: 'static + Copy + Eq + Hash;
1513
type Ident: 'static + Copy + Eq + Hash;
@@ -275,13 +273,17 @@ fn run_server<
275273
}
276274

277275
impl client::Client<crate::TokenStream, crate::TokenStream> {
278-
pub fn run<S: Server>(
276+
pub fn run<S>(
279277
&self,
280278
strategy: &impl ExecutionStrategy,
281279
server: S,
282280
input: S::TokenStream,
283281
force_show_panics: bool,
284-
) -> Result<S::TokenStream, PanicMessage> {
282+
) -> Result<S::TokenStream, PanicMessage>
283+
where
284+
S: Server,
285+
S::TokenStream: Default,
286+
{
285287
let client::Client { get_handle_counters, run, _marker } = *self;
286288
run_server(
287289
strategy,
@@ -291,19 +293,23 @@ impl client::Client<crate::TokenStream, crate::TokenStream> {
291293
run,
292294
force_show_panics,
293295
)
294-
.map(<MarkedTypes<S> as Types>::TokenStream::unmark)
296+
.map(|s| <Option<<MarkedTypes<S> as Types>::TokenStream>>::unmark(s).unwrap_or_default())
295297
}
296298
}
297299

298300
impl client::Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> {
299-
pub fn run<S: Server>(
301+
pub fn run<S>(
300302
&self,
301303
strategy: &impl ExecutionStrategy,
302304
server: S,
303305
input: S::TokenStream,
304306
input2: S::TokenStream,
305307
force_show_panics: bool,
306-
) -> Result<S::TokenStream, PanicMessage> {
308+
) -> Result<S::TokenStream, PanicMessage>
309+
where
310+
S: Server,
311+
S::TokenStream: Default,
312+
{
307313
let client::Client { get_handle_counters, run, _marker } = *self;
308314
run_server(
309315
strategy,
@@ -316,6 +322,6 @@ impl client::Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream
316322
run,
317323
force_show_panics,
318324
)
319-
.map(<MarkedTypes<S> as Types>::TokenStream::unmark)
325+
.map(|s| <Option<<MarkedTypes<S> as Types>::TokenStream>>::unmark(s).unwrap_or_default())
320326
}
321327
}

0 commit comments

Comments
 (0)