Trim trailing commas from semantic tokens output to ensure valid JSON (#7750)

* Add semantic tokens snapshot test

* Check not_compiled analysis test snapshots

* Trim trailing commas from semantic tokens output to ensure valid JSON

* Add CHANGELOG entry

Author: mediremi

CHANGELOG.md

@@ -22,6 +22,7 @@
 
 - Fix formatting of nested records in `.resi` files. https://github.com/rescript-lang/rescript/pull/7741
 - Don't format and don't check formatting of dependencies. https://github.com/rescript-lang/rescript/pull/7748
+- Fix `rescript-editor-analysis semanticTokens` returning invalid JSON in certain cases. https://github.com/rescript-lang/rescript/pull/7750
 
 #### :memo: Documentation
 

analysis/src/Commands.ml

@@ -498,6 +498,7 @@ let test ~path =
             let currentFile = createCurrentFile () in
             DumpAst.dump ~pos:(line, col) ~currentFile;
             Sys.remove currentFile
+          | "sem" -> SemanticTokens.semanticTokens ~currentFile:path
           | _ -> ());
           print_newline ())
     in

analysis/src/SemanticTokens.ml

@@ -79,6 +79,11 @@ module Token = struct
        ^ string_of_int length ^ "," ^ tokenTypeToString type_ ^ ","
        ^ tokenModifiersString)
 
+  let remove_trailing_comma buffer =
+    let len = Buffer.length buffer in
+    if len > 0 && Buffer.nth buffer (len - 1) = ',' then
+      Buffer.truncate buffer (len - 1)
+
   let emit e =
     let sortedTokens =
       e.tokens
@@ -87,6 +92,10 @@ module Token = struct
     in
     let buf = Buffer.create 1 in
     sortedTokens |> List.iter (fun t -> e |> emitToken buf t);
+
+    (* Valid JSON arrays cannot have trailing commas *)
+    remove_trailing_comma buf;
+
     Buffer.contents buf
 end
 

tests/analysis_tests/tests/not_compiled/SemanticTokens.res

@@ -0,0 +1,4 @@
+let make = () => {
+  <span className={ />
+}
+//^sem

tests/analysis_tests/tests/not_compiled/expected/SemanticTokens.res.txt

@@ -0,0 +1 @@
+{"data":[0,4,4,1,0,1,2,1,3,0,0,1,4,7,0]}

tests/analysis_tests/tests/test.sh

@@ -20,11 +20,11 @@ warningYellow='\033[0;33m'
 successGreen='\033[0;32m'
 reset='\033[0m'
 
-diff=$(git ls-files --modified src/expected)
+diff=$(git ls-files --modified src/expected not_compiled/expected)
 if [[ $diff = "" ]]; then
   printf "${successGreen}✅ No analysis_tests snapshot changes detected.${reset}\n"
 else
   printf "${warningYellow}⚠️ The analysis_tests snapshot doesn't match. Double check that the output is correct, run 'make test-analysis' and stage the diff.\n${diff}\n${reset}"
-  git --no-pager diff src/expected
+  git --no-pager diff src/expected not_compiled/expected
   exit 1
 fi