1313//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
1414//! ownership of the original.
1515
16- use crate :: ast:: StmtKind ;
16+ use crate :: ast:: { AttrStyle , StmtKind } ;
1717use crate :: ast_traits:: { HasAttrs , HasSpan , HasTokens } ;
1818use crate :: token:: { self , Delimiter , Nonterminal , Token , TokenKind } ;
1919use crate :: AttrVec ;
@@ -22,11 +22,11 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
2222use rustc_data_structures:: sync:: { self , Lrc } ;
2323use rustc_macros:: HashStable_Generic ;
2424use rustc_serialize:: { Decodable , Decoder , Encodable , Encoder } ;
25- use rustc_span:: { Span , DUMMY_SP } ;
25+ use rustc_span:: { sym , Span , Symbol , DUMMY_SP } ;
2626use smallvec:: { smallvec, SmallVec } ;
2727
2828use std:: borrow:: Cow ;
29- use std:: { fmt, iter, mem} ;
29+ use std:: { cmp , fmt, iter, mem} ;
3030
3131/// When the main Rust parser encounters a syntax-extension invocation, it
3232/// parses the arguments to the invocation as a token tree. This is a very
@@ -566,6 +566,92 @@ impl TokenStream {
566566 pub fn chunks ( & self , chunk_size : usize ) -> core:: slice:: Chunks < ' _ , TokenTree > {
567567 self . 0 . chunks ( chunk_size)
568568 }
569+
570+ /// Desugar doc comments like `/// foo` in the stream into `#[doc =
571+ /// r"foo"]`. Modifies the `TokenStream` via `Lrc::make_mut`, but as little
572+ /// as possible.
573+ pub fn desugar_doc_comments ( & mut self ) {
574+ if let Some ( desugared_stream) = desugar_inner ( self . clone ( ) ) {
575+ * self = desugared_stream;
576+ }
577+
578+ // The return value is `None` if nothing in `stream` changed.
579+ fn desugar_inner ( mut stream : TokenStream ) -> Option < TokenStream > {
580+ let mut i = 0 ;
581+ let mut modified = false ;
582+ while let Some ( tt) = stream. 0 . get ( i) {
583+ match tt {
584+ & TokenTree :: Token (
585+ Token { kind : token:: DocComment ( _, attr_style, data) , span } ,
586+ _spacing,
587+ ) => {
588+ let desugared = desugared_tts ( attr_style, data, span) ;
589+ let desugared_len = desugared. len ( ) ;
590+ Lrc :: make_mut ( & mut stream. 0 ) . splice ( i..i + 1 , desugared) ;
591+ modified = true ;
592+ i += desugared_len;
593+ }
594+
595+ & TokenTree :: Token ( ..) => i += 1 ,
596+
597+ & TokenTree :: Delimited ( sp, delim, ref delim_stream) => {
598+ if let Some ( desugared_delim_stream) = desugar_inner ( delim_stream. clone ( ) ) {
599+ let new_tt = TokenTree :: Delimited ( sp, delim, desugared_delim_stream) ;
600+ Lrc :: make_mut ( & mut stream. 0 ) [ i] = new_tt;
601+ modified = true ;
602+ }
603+ i += 1 ;
604+ }
605+ }
606+ }
607+ if modified { Some ( stream) } else { None }
608+ }
609+
610+ fn desugared_tts ( attr_style : AttrStyle , data : Symbol , span : Span ) -> Vec < TokenTree > {
611+ // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
612+ // required to wrap the text. E.g.
613+ // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
614+ // - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
615+ // - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
616+ let mut num_of_hashes = 0 ;
617+ let mut count = 0 ;
618+ for ch in data. as_str ( ) . chars ( ) {
619+ count = match ch {
620+ '"' => 1 ,
621+ '#' if count > 0 => count + 1 ,
622+ _ => 0 ,
623+ } ;
624+ num_of_hashes = cmp:: max ( num_of_hashes, count) ;
625+ }
626+
627+ // `/// foo` becomes `doc = r"foo"`.
628+ let delim_span = DelimSpan :: from_single ( span) ;
629+ let body = TokenTree :: Delimited (
630+ delim_span,
631+ Delimiter :: Bracket ,
632+ [
633+ TokenTree :: token_alone ( token:: Ident ( sym:: doc, false ) , span) ,
634+ TokenTree :: token_alone ( token:: Eq , span) ,
635+ TokenTree :: token_alone (
636+ TokenKind :: lit ( token:: StrRaw ( num_of_hashes) , data, None ) ,
637+ span,
638+ ) ,
639+ ]
640+ . into_iter ( )
641+ . collect :: < TokenStream > ( ) ,
642+ ) ;
643+
644+ if attr_style == AttrStyle :: Inner {
645+ vec ! [
646+ TokenTree :: token_alone( token:: Pound , span) ,
647+ TokenTree :: token_alone( token:: Not , span) ,
648+ body,
649+ ]
650+ } else {
651+ vec ! [ TokenTree :: token_alone( token:: Pound , span) , body]
652+ }
653+ }
654+ }
569655}
570656
571657/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
@@ -628,15 +714,6 @@ impl TokenTreeCursor {
628714 pub fn look_ahead ( & self , n : usize ) -> Option < & TokenTree > {
629715 self . stream . 0 . get ( self . index + n)
630716 }
631-
632- // Replace the previously obtained token tree with `tts`, and rewind to
633- // just before them.
634- pub fn replace_prev_and_rewind ( & mut self , tts : Vec < TokenTree > ) {
635- assert ! ( self . index > 0 ) ;
636- self . index -= 1 ;
637- let stream = Lrc :: make_mut ( & mut self . stream . 0 ) ;
638- stream. splice ( self . index ..self . index + 1 , tts) ;
639- }
640717}
641718
642719#[ derive( Debug , Copy , Clone , PartialEq , Encodable , Decodable , HashStable_Generic ) ]
0 commit comments