use externals for comments

done for the sake of differentiating the next character after '//', which requires forward lookup, and could not be achieved via regular expressions

Author: resolritter

grammar.js

@@ -45,6 +45,8 @@ module.exports = grammar({
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
+    $.line_comment,
+    $.doc_comment
   ],
 
   supertypes: $ => [
@@ -1410,14 +1412,6 @@ module.exports = grammar({
 
     boolean_literal: $ => choice('true', 'false'),
 
-    doc_comment: $ => token(seq(
-      '///', /.*/
-    )),
-
-    line_comment: $ => token(seq(
-      new RegExp('\/\/[^\/]'), /.*/
-    )),
-
     _path: $ => choice(
       $.self,
       alias(choice(...primitive_types), $.identifier),

src/scanner.c

@@ -6,6 +6,8 @@ enum TokenType {
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
+  LINE_COMMENT,
+  DOC_COMMENT,
 };
 
 void *tree_sitter_rust_external_scanner_create() { return NULL; }
@@ -143,7 +145,23 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
 
   if (lexer->lookahead == '/') {
     advance(lexer);
+
+    if ((valid_symbols[LINE_COMMENT] || valid_symbols[DOC_COMMENT]) && lexer->lookahead == '/') {
+      advance(lexer);
+      if (lexer->lookahead == '/') {
+        lexer->result_symbol = DOC_COMMENT;
+      } else {
+        lexer->result_symbol = LINE_COMMENT;
+      }
+
+      while (lexer->lookahead != '\n' && lexer->lookahead != 0) {
+        advance(lexer);
+      }
+      return true;
+    }
+
     if (lexer->lookahead != '*') return false;
+
     advance(lexer);
 
     bool after_star = false;