designate doc comments

Author: resolritter

corpus/source_files.txt

@@ -55,6 +55,36 @@ Line comments
 (source_file
   (line_comment))
 
+============================================
+Doc comments
+============================================
+
+/// Doc
+/// Comment
+// / Now a line comment (note the space separating the third slash)
+
+/// Doc
+/// Comment
+//// Four slashes makes the line a normal comment
+/// Doc comment got interrupted by the line above
+
+//! Inner doc comment line 1
+//! Inner doc comment line 2
+/// This is different doc comment since the line starts differently
+//! Back to inner doc comment
+
+----
+
+(source_file
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment))
+
 =====================================
 Greek letters in identifiers
 =====================================

grammar.js

@@ -38,13 +38,15 @@ const primitive_types = numeric_types.concat(['bool', 'str', 'char'])
 module.exports = grammar({
   name: 'rust',
 
-  extras: $ => [/\s/, $.line_comment, $.block_comment],
+  extras: $ => [/\s/, $.line_comment, $.block_comment, $.doc_comment],
 
   externals: $ => [
     $._string_content,
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
+    $.line_comment,
+    $.doc_comment
   ],
 
   supertypes: $ => [
@@ -1426,15 +1428,6 @@ module.exports = grammar({
 
     boolean_literal: $ => choice('true', 'false'),
 
-    comment: $ => choice(
-      $.line_comment,
-      $.block_comment
-    ),
-
-    line_comment: $ => token(seq(
-      '//', /.*/
-    )),
-
     _path: $ => choice(
       $.self,
       alias(choice(...primitive_types), $.identifier),

src/scanner.c

@@ -6,6 +6,8 @@ enum TokenType {
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
+  LINE_COMMENT,
+  DOC_COMMENT,
 };
 
 void *tree_sitter_rust_external_scanner_create() { return NULL; }
@@ -143,7 +145,76 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
 
   if (lexer->lookahead == '/') {
     advance(lexer);
+
+    if ((valid_symbols[LINE_COMMENT] || valid_symbols[DOC_COMMENT]) && lexer->lookahead == '/') {
+      advance(lexer);
+
+      bool started_with_slash = lexer->lookahead == '/';
+      switch (lexer->lookahead) {
+        case '!':
+        case '/': {
+          advance(lexer);
+
+          // If three consecutive slashes were seen and this is the fourth one,
+          // the line turns back to a normal comment.
+          // The above rule does not apply for "//!" which is also a doc
+          // comment, hence why it is relevant to track started_with_slash.
+          if (started_with_slash == false || lexer->lookahead != '/') {
+            lexer->result_symbol = DOC_COMMENT;
+            while (true) {
+              while (lexer->lookahead != '\n') {
+                advance(lexer);
+              }
+              if (lexer->lookahead == 0) {
+                break;
+              }
+
+              lexer->mark_end(lexer);
+              advance(lexer);
+              if (lexer->lookahead == '/') {
+                advance(lexer);
+                if (lexer->lookahead == '/') {
+                  advance(lexer);
+                  if (started_with_slash) {
+                    if (lexer->lookahead == '/') {
+                      advance(lexer);
+                      // If a fourth slash is found, the line turns back to a normal comment
+                      if (lexer->lookahead == '/') {
+                        break;
+                      }
+                    } else {
+                      break;
+                    }
+                  } else if (lexer->lookahead != '!') {
+                    break;
+                  }
+                } else {
+                  break;
+                }
+              } else {
+                break;
+              }
+            }
+          }
+          break;
+        }
+      }
+
+
+      // Might have broken from the loop above having already processed a doc
+      // comment
+      if (lexer->result_symbol != DOC_COMMENT) {
+        lexer->result_symbol = LINE_COMMENT;
+        while (lexer->lookahead != '\n' && lexer->lookahead != 0) {
+          advance(lexer);
+        }
+      }
+
+      return true;
+    }
+
     if (lexer->lookahead != '*') return false;
+
     advance(lexer);
 
     bool after_star = false;