Add some comments for magic numbers + Add tests

Author: petrochenkov

src/librustc_parse/lexer/mod.rs

@@ -172,6 +172,7 @@ impl<'a> StringReader<'a> {
                 let string = self.str_from(start);
                 if let Some(attr_style) = comments::line_doc_comment_style(string) {
                     self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment");
+                    // Opening delimiter of the length 3 is not included into the symbol.
                     token::DocComment(CommentKind::Line, attr_style, Symbol::intern(&string[3..]))
                 } else {
                     token::Comment
@@ -201,6 +202,8 @@ impl<'a> StringReader<'a> {
 
                 if let Some(attr_style) = attr_style {
                     self.forbid_bare_cr(start, string, "bare CR not allowed in block doc-comment");
+                    // Opening delimiter of the length 3 and closing delimiter of the length 2
+                    // are not included into the symbol.
                     token::DocComment(
                         CommentKind::Block,
                         attr_style,

src/test/ui/proc-macro/doc-comment-preserved.rs

@@ -0,0 +1,24 @@
+// check-pass
+// aux-build:test-macros.rs
+
+// Anonymize unstable non-dummy spans while still showing dummy spans `0..0`.
+// normalize-stdout-test "bytes\([^0]\w*\.\.(\w+)\)" -> "bytes(LO..$1)"
+// normalize-stdout-test "bytes\((\w+)\.\.[^0]\w*\)" -> "bytes($1..HI)"
+
+#[macro_use]
+extern crate test_macros;
+
+print_bang! {
+
+/**
+*******
+* DOC *
+* DOC *
+* DOC *
+*******
+*/
+pub struct S;
+
+}
+
+fn main() {}

src/test/ui/proc-macro/doc-comment-preserved.stdout

@@ -0,0 +1,54 @@
+PRINT-BANG INPUT (DISPLAY): /**
+*******
+* DOC *
+* DOC *
+* DOC *
+*******
+*/
+ pub struct S ;
+PRINT-BANG RE-COLLECTED (DISPLAY): #[doc = "\n*******\n* DOC *\n* DOC *\n* DOC *\n*******\n"] pub struct S ;
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Punct {
+        ch: '#',
+        spacing: Alone,
+        span: #0 bytes(LO..HI),
+    },
+    Group {
+        delimiter: Bracket,
+        stream: TokenStream [
+            Ident {
+                ident: "doc",
+                span: #0 bytes(LO..HI),
+            },
+            Punct {
+                ch: '=',
+                spacing: Alone,
+                span: #0 bytes(LO..HI),
+            },
+            Literal {
+                kind: Str,
+                symbol: "\n*******\n* DOC *\n* DOC *\n* DOC *\n*******\n",
+                suffix: None,
+                span: #0 bytes(LO..HI),
+            },
+        ],
+        span: #0 bytes(LO..HI),
+    },
+    Ident {
+        ident: "pub",
+        span: #0 bytes(LO..HI),
+    },
+    Ident {
+        ident: "struct",
+        span: #0 bytes(LO..HI),
+    },
+    Ident {
+        ident: "S",
+        span: #0 bytes(LO..HI),
+    },
+    Punct {
+        ch: ';',
+        spacing: Alone,
+        span: #0 bytes(LO..HI),
+    },
+]

src/tools/clippy/clippy_lints/src/doc.rs

@@ -264,6 +264,7 @@ pub fn strip_doc_comment_decoration(doc: &str, comment_kind: CommentKind, span:
         let mut doc = doc.to_owned();
         doc.push('\n');
         let len = doc.len();
+        // +3 skips the opening delimiter
         return (doc, vec![(len, span.with_lo(span.lo() + BytePos(3)))]);
     }
 
@@ -273,7 +274,7 @@ pub fn strip_doc_comment_decoration(doc: &str, comment_kind: CommentKind, span:
         let offset = line.as_ptr() as usize - doc.as_ptr() as usize;
         debug_assert_eq!(offset as u32 as usize, offset);
         contains_initial_stars |= line.trim_start().starts_with('*');
-        // +1 for the newline
+        // +1 adds the newline, +3 skips the opening delimiter
         sizes.push((line.len() + 1, span.with_lo(span.lo() + BytePos(3 + offset as u32))));
     }
     if !contains_initial_stars {

src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs

@@ -64,6 +64,7 @@ impl TabsInDocComments {
             let comment = comment.as_str();
 
             for (lo, hi) in get_chunks_of_tabs(&comment) {
+                // +3 skips the opening delimiter
                 let new_span = Span::new(
                     attr.span.lo() + BytePos(3 + lo),
                     attr.span.lo() + BytePos(3 + hi),