replace logos with nom

nom is more lightweight and more suitable for this task

Author: gerau

Cargo.toml

@@ -16,7 +16,7 @@ thiserror = "2.0.17"
 ropey = "1.6.1"
 miniscript = "12"
 simplicityhl = { git = "https://github.com/BlockstreamResearch/SimplicityHL.git", rev = "e68e1c6" }
-logos = "0.15.1"
+nom = "8.0.0"
 
 [lints.rust]
 unsafe_code = "deny"

src/completion/mod.rs

@@ -1,4 +1,3 @@
-use logos::Logos;
 use simplicityhl::parse::Function;
 
 pub mod builtin;
@@ -12,6 +11,7 @@ use tower_lsp_server::lsp_types::{
 };
 
 use tokens::Token;
+use tokens::lex_tokens;
 
 /// Build and provide [`CompletionItem`] for jets and builtin functions.
 #[derive(Debug)]
@@ -88,18 +88,33 @@ impl CompletionProvider {
         prefix: &str,
         functions: &[(&Function, &str)],
     ) -> Option<Vec<CompletionItem>> {
-        let mut tokens: Vec<Token> = Token::lexer(prefix).filter_map(Result::ok).collect();
-        tokens.reverse();
+        let tokens = match lex_tokens(prefix) {
+            Ok((_, mut t)) => {
+                t.reverse();
+                t
+            }
+            Err(_) => return None,
+        };
 
         match tokens.as_slice() {
             [Token::Jet, ..] => Some(self.jets.clone()),
 
+            // Case for ": type = <", so we can return completion for specific type, or generic one
+            // if it is not on default type casts.
             [
                 Token::OpenAngle,
                 Token::EqualSign,
                 Token::Identifier(type_name),
                 Token::Colon,
                 ..,
+            ]
+            | [
+                Token::Identifier(_) | Token::OpenBracket,
+                Token::OpenAngle,
+                Token::EqualSign,
+                Token::Identifier(type_name),
+                Token::Colon,
+                ..,
             ] => {
                 let to = type_name.as_str();
 
@@ -117,6 +132,7 @@ impl CompletionProvider {
                 Some(self.type_casts.clone())
             }
 
+            // Case for ">::" -- this structure is only present for into keyword.
             [Token::DoubleColon, Token::CloseAngle, ..] => Some(vec![CompletionItem {
                 label: "into".to_string(),
                 kind: Some(CompletionItemKind::FUNCTION),
@@ -131,11 +147,6 @@ impl CompletionProvider {
 
             _ => {
                 let mut completions = CompletionProvider::get_function_completions(functions);
-                // return only function completions in case of '<' symbol in `for_while`, `array_fold` and
-                // `fold`
-                if prefix.ends_with('<') {
-                    return Some(completions);
-                }
 
                 completions.extend_from_slice(&self.builtin);
                 completions.extend_from_slice(&self.modules);

src/completion/tokens.rs

@@ -1,28 +1,65 @@
-use logos::Logos;
+use nom::{
+    IResult, Parser,
+    branch::alt,
+    bytes::complete::{tag, take_while},
+    character::complete::{multispace0, satisfy},
+    combinator::{map, opt, recognize, value},
+    multi::many0,
+    sequence::{pair, preceded},
+};
 
-#[derive(Logos, Debug, PartialEq, Clone)]
-#[logos(skip r"[ \t\r\n\f]+")]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Token {
-    #[token(":")]
     Colon,
-
-    #[token("::")]
     DoubleColon,
-
-    #[token("<")]
     OpenAngle,
-
-    #[token(">")]
     CloseAngle,
-
-    #[token("=")]
     EqualSign,
-
-    #[regex(r"\(\s*[a-zA-Z_][a-zA-Z0-9_]*\s*,\s*[a-zA-Z_][a-zA-Z0-9_]*\s*\)", |lex| lex.slice().to_string())]
-    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_string())]
+    OpenBracket,
+    ClosedBracket,
     Identifier(String),
-
-    #[regex(r"jet::[a-zA-Z0-9_]?", priority = 2)]
-    #[token("jet::", priority = 1)]
     Jet,
 }
+
+fn parse_symbol(input: &str) -> IResult<&str, Token> {
+    let mut parser = alt((
+        value(Token::DoubleColon, tag("::")),
+        value(Token::Colon, tag(":")),
+        value(Token::OpenBracket, tag("(")),
+        value(Token::ClosedBracket, tag(")")),
+        value(Token::OpenAngle, tag("<")),
+        value(Token::CloseAngle, tag(">")),
+        value(Token::EqualSign, tag("=")),
+    ));
+    parser.parse(input)
+}
+
+fn parse_jet(input: &str) -> IResult<&str, Token> {
+    let mut parser = value(
+        Token::Jet,
+        recognize(pair(
+            tag("jet::"),
+            opt(take_while(|c: char| c.is_alphanumeric() || c == '_')),
+        )),
+    );
+    parser.parse(input)
+}
+
+fn parse_identifier(input: &str) -> IResult<&str, Token> {
+    let mut parser = map(
+        recognize(pair(
+            satisfy(|c| c.is_alphabetic() || c == '_'),
+            take_while(|c: char| c.is_alphanumeric() || c == '_'),
+        )),
+        |s: &str| Token::Identifier(s.to_string()),
+    );
+    parser.parse(input)
+}
+
+pub fn lex_tokens(input: &str) -> IResult<&str, Vec<Token>> {
+    let mut parser = many0(preceded(
+        multispace0,
+        alt((parse_jet, parse_symbol, parse_identifier)),
+    ));
+    parser.parse(input)
+}