Fix zoekt language parsing and add more token coloring (#115)

Author: KonradStanski

packages/web/src/app/components/searchBar/searchBar.tsx

@@ -144,10 +144,18 @@ export const SearchBar = ({
                     tag: t.keyword,
                     color: tailwind.theme.colors.highlight,
                 },
+                {
+                    tag: t.string,
+                    color: '#2aa198',
+                },
+                {
+                    tag: t.operator,
+                    color: '#d33682',
+                },
                 {
                     tag: t.paren,
                     color: tailwind.theme.colors.highlight,
-                }
+                },
             ],
         });
     }, [tailwind]);
@@ -333,4 +341,4 @@ const SearchHistoryButton = ({
             </TooltipContent>
         </Tooltip>
     )
-}
+}

packages/web/src/app/components/searchBar/zoektLanguageExtension.ts

@@ -1,25 +1,78 @@
 import { LanguageSupport, StreamLanguage } from "@codemirror/language";
-import { tags as t } from '@lezer/highlight';
+import { tags as t } from "@lezer/highlight";
 
-const zoektLanguage = StreamLanguage.define({
-    token: (stream) => {
-        if (stream.match(/-?(file|branch|revision|rev|case|repo|lang|content|sym|archived|fork|public):/)) {
-            return t.keyword.toString();
-        }
+export const zoekt = () => {
+    const zoektLanguage = StreamLanguage.define({
+        startState() {
+            return {
+                inString: false,
+                escaped: false
+            };
+        },
+        token(stream, state) {
+            // Handle strings
+            if (state.inString) {
+                if (state.escaped) {
+                    state.escaped = false;
+                    stream.next();
+                    return t.string.toString();
+                }
+                const ch = stream.next();
+                if (ch === "\\") {
+                    state.escaped = true;
+                    return t.string.toString();
+                } else if (ch === '"') {
+                    // End of string
+                    state.inString = false;
+                    return t.string.toString();
+                } else {
+                    return t.string.toString();
+                }
+            }
 
-        if (stream.match(/\bor\b/)) {
-            return t.keyword.toString();
-        }
+            // Skip whitespace
+            if (stream.eatSpace()) {
+                return null;
+            }
 
-        if (stream.match(/(\(|\))/)) {
-            return t.paren.toString();
-        }
+            // Negation operator
+            if (stream.match(/-/)) {
+                return t.operator.toString();
+            }
 
-        stream.next();
-        return null;
-    },
-});
+            // Parentheses
+            if (stream.match("(") || stream.match(")")) {
+                return t.paren.toString();
+            }
+
+            // Check for prefixes first
+            // If these match, we return 'keyword'
+            if (stream.match(/(archived:|branch:|b:|c:|case:|content:|f:|file:|fork:|public:|r:|repo:|regex:|lang:|sym:|t:|type:)/)) {
+                return t.keyword.toString();
+            }
+
+            // Now try matching a standalone word
+            // If the word is "or", return keyword; else nothing special
+            if (stream.match(/[A-Za-z0-9_]+/)) {
+                const word = stream.current();
+                if (word === "or") {
+                    return t.keyword.toString();
+                }
+                return null;
+            }
+
+            // Double-quoted string start
+            if (stream.peek() === '"') {
+                stream.next(); // consume opening quote
+                state.inString = true;
+                return t.string.toString();
+            }
+
+            // If we reach here, consume a single character and return null
+            stream.next();
+            return null;
+        }
+    });
 
-export const zoekt = () => {
     return new LanguageSupport(zoektLanguage);
-}
+};