Refactor jsx mode in parser (#7751)

* Add token dump printer

* Remove ForwardSlash token

* Document RESCRIPT_BSC_EXE for local usage

* Process hypens in parse_module_long_ident for jsx

* Update test snapshot

* Add how to view tokens.

* fmt

* Clean up

* Correct fragment range

* Use legacy clean for analysis projects

* Update analysis snapshot

* Copilot review suggestion

* Return original error directly

* Add trailing hypen error

* Mutate token on the call side.

* Add changelog entry

Author: nojaf

CHANGELOG.md

@@ -32,6 +32,9 @@
 
 #### :house: Internal
 
+- Add token viewer to `res_parser`. https://github.com/rescript-lang/rescript/pull/7751
+- Refactor jsx mode in Scanner. https://github.com/rescript-lang/rescript/pull/7751
+
 # 12.0.0-beta.4
 
 #### :bug: Bug fix

CONTRIBUTING.md

@@ -126,12 +126,24 @@ make lib # Build compiler and standard library
 ./cli/bsc.js myTestFile.res
 ```
 
+To view the tokens of a file run:
+
+```sh
+dune exec res_parser -- -print tokens myTestFile.res
+```
+
 To view the untyped tree of the file run:
 
 ```sh
 ./cli/bsc.js -dparsetree myTestFile.res
 ```
 
+or
+
+```sh
+dune exec res_parser -- -print ast -recover myTestFile.res
+```
+
 To view the typed tree of the file run:
 
 ```sh
@@ -148,6 +160,16 @@ npm install
 npm link rescript
 ```
 
+#### Use Local BSC with Existing ReScript Installation
+
+Alternatively, you can set the `RESCRIPT_BSC_EXE` environment variable to point to your locally compiled `bsc.exe`.
+
+```sh
+RESCRIPT_BSC_EXE=your-rescript-repo/packages/@rescript/darwin-arm64/bin/bsc.exe npx rescript
+```
+
+This will test the local compiler while still using the build system from the installed Node module.
+
 ### Running Automatic Tests
 
 We provide different test suites for different levels of the compiler and build system infrastructure. Always make sure to locally build your compiler before running any tests.

compiler/syntax/cli/res_cli.ml

@@ -194,7 +194,8 @@ end = struct
       ("-recover", Arg.Unit (fun () -> recover := true), "Emit partial ast");
       ( "-print",
         Arg.String (fun txt -> print := txt),
-        "Print either binary, ml, ast, sexp, comments or res. Default: res" );
+        "Print either binary, ml, ast, sexp, comments, tokens or res. Default: \
+         res" );
       ( "-width",
         Arg.Int (fun w -> width := w),
         "Specify the line length for the printer (formatter)" );
@@ -239,11 +240,12 @@ module CliArgProcessor = struct
       | "ast" -> Res_ast_debugger.print_engine
       | "sexp" -> Res_ast_debugger.sexp_print_engine
       | "comments" -> Res_ast_debugger.comments_print_engine
+      | "tokens" -> Res_token_debugger.token_print_engine
       | "res" -> Res_driver.print_engine
       | target ->
         print_endline
-          ("-print needs to be either binary, ml, ast, sexp, comments or res. \
-            You provided " ^ target);
+          ("-print needs to be either binary, ml, ast, sexp, comments, tokens \
+            or res. You provided " ^ target);
         exit 1
     in
 
@@ -256,7 +258,11 @@ module CliArgProcessor = struct
     let (Parser backend) = parsing_engine in
     (* This is the whole purpose of the Color module above *)
     Color.setup None;
-    if process_interface then
+
+    (* Special case for tokens - bypass parsing entirely *)
+    if target = "tokens" then
+      print_engine.print_implementation ~width ~filename ~comments:[] []
+    else if process_interface then
       let parse_result = backend.parse_interface ~for_printer ~filename in
       if parse_result.invalid then (
         backend.string_of_diagnostics ~source:parse_result.source