designate doc comments

Author: resolritter

corpus/source_files.txt

@@ -55,6 +55,86 @@ Line comments
 (source_file
   (line_comment))
 
+============================================
+Doc comments
+============================================
+
+/// Doc
+/// Comment
+// / Now a line comment (note the space separating the third slash)
+
+/// Doc
+/// Comment
+//// Four slashes makes the line a normal comment
+/// Doc comment got interrupted by the line above
+
+//! Inner doc comment line 1
+//! Inner doc comment line 2
+/// This is different doc comment since the line starts differently
+//! Back to inner doc comment
+
+  /// Same doc comment
+    /// But with arbitrary indentation
+
+/// Different doc comments
+
+/// Are separated by newlines
+
+----
+
+(source_file
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (line_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment)
+  (doc_comment))
+
+=================================================
+Doc comments recursion guard 1 (should not hang)
+=================================================
+
+//! 
+
+---
+
+(source_file (doc_comment))
+
+=================================================
+Doc comments recursion guard 2 (should not hang)
+=================================================
+
+///
+
+---
+
+(source_file (doc_comment))
+
+=================================================
+Comments recursion guard 1 (should not hang)
+=================================================
+
+//
+
+---
+
+(source_file (line_comment))
+
+=================================================
+Block comments recursion guard 1 (should not hang)
+=================================================
+
+/*
+
+---
+
+(source_file (ERROR))
+
 =====================================
 Greek letters in identifiers
 =====================================

grammar.js

@@ -38,13 +38,15 @@ const primitive_types = numeric_types.concat(['bool', 'str', 'char'])
 module.exports = grammar({
   name: 'rust',
 
-  extras: $ => [/\s/, $.line_comment, $.block_comment],
+  extras: $ => [/\s/, $.line_comment, $.block_comment, $.doc_comment],
 
   externals: $ => [
     $._string_content,
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
+    $.line_comment,
+    $.doc_comment
   ],
 
   supertypes: $ => [
@@ -1426,15 +1428,6 @@ module.exports = grammar({
 
     boolean_literal: $ => choice('true', 'false'),
 
-    comment: $ => choice(
-      $.line_comment,
-      $.block_comment
-    ),
-
-    line_comment: $ => token(seq(
-      '//', /.*/
-    )),
-
     _path: $ => choice(
       $.self,
       alias(choice(...primitive_types), $.identifier),

src/scanner.c

@@ -6,6 +6,8 @@ enum TokenType {
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
+  LINE_COMMENT,
+  DOC_COMMENT,
 };
 
 void *tree_sitter_rust_external_scanner_create() { return NULL; }
@@ -143,7 +145,95 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
 
   if (lexer->lookahead == '/') {
     advance(lexer);
+
+    if ((valid_symbols[LINE_COMMENT] || valid_symbols[DOC_COMMENT]) && lexer->lookahead == '/') {
+      advance(lexer);
+
+      bool started_with_slash = lexer->lookahead == '/';
+      switch (lexer->lookahead) {
+        case '!':
+        case '/': {
+          advance(lexer);
+
+          // If three consecutive slashes were seen and this is the fourth one,
+          // the line turns back to a normal comment.
+          // The above rule does not apply for "//!" which is also a doc
+          // comment, hence why it is relevant to track started_with_slash.
+          if (started_with_slash == false || lexer->lookahead != '/') {
+            lexer->result_symbol = DOC_COMMENT;
+
+            while (true) {
+              while (true) {
+                switch (lexer->lookahead) {
+                  case '\n': {
+                    lexer->mark_end(lexer);
+                    advance(lexer);
+                    goto finished_doc_comment_line;
+                  }
+                  case 0: {
+                    goto doc_comment_exit;
+                  }
+                  default: {
+                    advance(lexer);
+                  }
+                }
+              }
+
+finished_doc_comment_line:
+
+              while (isblank(lexer->lookahead)) lexer->advance(lexer, false);
+
+              if (lexer->lookahead == '\n') {
+                // Even if there's another comment ahead, it'll be part of a
+                // separate node. Break here.
+                break;
+              }
+
+              if (lexer->lookahead == '/') {
+                advance(lexer);
+                if (lexer->lookahead == '/') {
+                  advance(lexer);
+                  if (started_with_slash) {
+                    if (lexer->lookahead == '/') {
+                      advance(lexer);
+                      // If a fourth slash is found, the line turns back to a normal comment
+                      if (lexer->lookahead == '/') {
+                        break;
+                      }
+                    } else {
+                      break;
+                    }
+                  } else if (lexer->lookahead != '!') {
+                    break;
+                  }
+                } else {
+                  break;
+                }
+              } else {
+                break;
+              }
+            }
+          }
+
+          break;
+        }
+      }
+
+doc_comment_exit:
+
+      // Might have already processed a doc comment in the loop above
+      if (lexer->result_symbol != DOC_COMMENT) {
+        lexer->result_symbol = LINE_COMMENT;
+        while (lexer->lookahead != '\n' && lexer->lookahead != 0) {
+          advance(lexer);
+        }
+      }
+
+      return true;
+    }
+
     if (lexer->lookahead != '*') return false;
+
     advance(lexer);
 
     bool after_star = false;