Compile runtime with rewatch and add rewatch tests to the compiler repo (#7422)

* change config

* still need -make-runtime

* add make lib-rewatch

* update rewatch

* test rewatch in Makefile

* test

* add install state

* upstream new version of rewatch

* make it work!

* make rewatch default for building the runtime

* run rewatch tests

* see error

* install packages as part of the ci test suite

* cat the changes

* empty commit

* immutable install

* remove install state

* install state ignore

* run on mac os due to subtle diffs

* fix

* use ci version of rescript

* fix diff

* trigger

* fix

* show rescript version

* update snapshots

* end to end tests

* fix

* fix

* fix

* propagate bsc path

* fix

* empty commit

* don't run unit tests if we don't build a new rewatch binary

* other instance

* empty commit

* find correct bsc

* if the lockfile is deleted, stop running the watcher

* fix lock tests

* More informative test

* fix watch

* make tests work correctly

* fix

* upstream changes

* test

* fix

* commit

* fix js

* fix diff

* remove deps

* add core dep

* fix lockfile

* Build runtime with rewatch in CI

* Fix CI, move rewatch build steps into main compiler build job

* Revert "Build runtime with rewatch in CI"

This reverts commit 64f84ea.

* Build runtime with rewatch unless on Windows

* Clean up integration test running

* Rewatch integration tests don't work on Windows and macOS x64

* CHANGELOG

* Run rewatch integration tests on Linux only

---------

Co-authored-by: Christoph Knittel <ck@cca.io>

Author: jfrolich

.github/workflows/ci.yml

@@ -20,76 +20,6 @@ env:
   OCAMLRUNPARAM: b
 
 jobs:
-  build-rewatch:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: macos-13 # x64
-            rust-target: x86_64-apple-darwin
-            node-target: darwin-x64
-          - os: macos-14 # ARM
-            rust-target: aarch64-apple-darwin
-            node-target: darwin-arm64
-          - os: ubuntu-24.04 # x64
-            rust-target: x86_64-unknown-linux-musl
-            node-target: linux-x64
-          - os: ubuntu-24.04-arm # ARM
-            rust-target: aarch64-unknown-linux-musl
-            node-target: linux-arm64
-          - os: windows-latest
-            rust-target: x86_64-pc-windows-gnu
-            node-target: win32-x64
-
-    runs-on: ${{matrix.os}}
-
-    env:
-      RUST_BACKTRACE: "1"
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Use Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: .nvmrc
-
-      - name: Restore build cache
-        id: build-cache
-        uses: actions/cache@v4
-        with:
-          path: rewatch/target
-          key: rewatch-build-v2-${{ matrix.rust-target }}-${{ hashFiles('rewatch/src/**', 'rewatch/Cargo.lock') }}
-
-      - name: Install musl gcc
-        if: steps.build-cache.outputs.cache-hit != 'true' && runner.os == 'Linux'
-        run: sudo apt-get install -y --no-install-recommends musl-tools
-
-      - name: Install rust toolchain
-        if: steps.build-cache.outputs.cache-hit != 'true'
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: stable
-          targets: ${{ matrix.rust-target }}
-
-      - name: Build rewatch
-        if: steps.build-cache.outputs.cache-hit != 'true'
-        run: |
-          cargo build --manifest-path rewatch/Cargo.toml --target ${{ matrix.rust-target }} --release
-
-      - name: Copy rewatch binary
-        run: |
-          cp rewatch/target/${{ matrix.rust-target }}/release/rewatch${{ runner.os == 'Windows' && '.exe' || '' }} rewatch.exe
-        shell: bash
-
-      - name: "Upload artifact: rewatch binary"
-        uses: actions/upload-artifact@v4
-        with:
-          name: rewatch-${{ matrix.node-target }}
-          path: rewatch.exe
-          if-no-files-found: error
-
   build-compiler:
     strategy:
       fail-fast: false
@@ -100,33 +30,44 @@ jobs:
             upload_binaries: true
             upload_libs: true
             node-target: linux-x64
+            rust-target: x86_64-unknown-linux-musl
           - os: ubuntu-24.04-arm # ARM
             ocaml_compiler: ocaml-variants.5.3.0+options,ocaml-option-static
             upload_binaries: true
             # Build the playground compiler and run the benchmarks on the fastest runner
             build_playground: true
             benchmarks: true
             node-target: linux-arm64
+            rust-target: aarch64-unknown-linux-musl
           - os: macos-13 # x64
             ocaml_compiler: 5.3.0
             upload_binaries: true
             node-target: darwin-x64
+            rust-target: x86_64-apple-darwin
           - os: macos-14 # ARM
             ocaml_compiler: 5.3.0
             upload_binaries: true
             node-target: darwin-arm64
+            rust-target: aarch64-apple-darwin
           - os: windows-latest
             ocaml_compiler: 5.3.0
             upload_binaries: true
             node-target: win32-x64
+            rust-target: x86_64-pc-windows-gnu
 
           # Verify that the compiler still builds with older OCaml versions
           - os: ubuntu-24.04
             ocaml_compiler: ocaml-variants.5.2.1+options,ocaml-option-static
+            node-target: linux-x64
+            rust-target: x86_64-unknown-linux-musl
           - os: ubuntu-24.04
             ocaml_compiler: ocaml-variants.5.0.0+options,ocaml-option-static
+            node-target: linux-x64
+            rust-target: x86_64-unknown-linux-musl
           - os: ubuntu-24.04
             ocaml_compiler: ocaml-variants.4.14.2+options,ocaml-option-static
+            node-target: linux-x64
+            rust-target: x86_64-unknown-linux-musl
 
     runs-on: ${{matrix.os}}
 
@@ -135,6 +76,7 @@ jobs:
       # we do track its version manually
       OPAM_VERSION: 2.3.0
       DUNE_PROFILE: release
+      RUST_BACKTRACE: "1"
 
     steps:
       - name: "Windows: Set git config"
@@ -153,6 +95,9 @@ jobs:
           cache: yarn
           node-version-file: .nvmrc
 
+      - name: Install npm packages
+        run: yarn install
+
       - name: Install dependencies (Linux)
         if: runner.os == 'Linux'
         uses: awalsh128/cache-apt-pkgs-action@v1.4.3
@@ -161,6 +106,36 @@ jobs:
           packages: bubblewrap darcs g++-multilib gcc-multilib mercurial musl-tools rsync
           version: v3
 
+      - name: Restore rewatch build cache
+        id: rewatch-build-cache
+        uses: actions/cache@v4
+        with:
+          path: rewatch/target
+          key: rewatch-build-v2-${{ matrix.rust-target }}-${{ hashFiles('rewatch/src/**', 'rewatch/Cargo.lock') }}
+
+      - name: Install rust toolchain
+        if: steps.rewatch-build-cache.outputs.cache-hit != 'true'
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          targets: ${{ matrix.rust-target }}
+
+      - name: Build rewatch
+        if: steps.rewatch-build-cache.outputs.cache-hit != 'true'
+        run: |
+          cargo build --manifest-path rewatch/Cargo.toml --target ${{ matrix.rust-target }} --release
+
+      - name: Run rewatch unit tests
+        if: steps.rewatch-build-cache.outputs.cache-hit != 'true'
+        run: |
+          cargo test --manifest-path rewatch/Cargo.toml
+
+      - name: Copy rewatch binary
+        run: |
+          cp rewatch/target/${{ matrix.rust-target }}/release/rewatch${{ runner.os == 'Windows' && '.exe' || '' }} rewatch
+          ./scripts/copyExes.js --rewatch
+        shell: bash
+
       # matrix.ocaml_compiler may contain commas
       - name: Get OPAM cache key
         shell: bash
@@ -305,9 +280,6 @@ jobs:
             _build
           key: ${{ steps.compiler-build-state-key.outputs.value }}
 
-      - name: Install npm packages
-        run: yarn install
-
       - name: Copy compiler exes to platform bin dir
         run: node scripts/copyExes.js --compiler
 
@@ -352,7 +324,13 @@ jobs:
         if: ${{ runner.os == 'Windows' }}
         run: opam exec -- make test-syntax
 
-      - name: Build runtime/stdlib
+      - name: Build runtime/stdlib with rewatch
+        if: ${{ runner.os != 'Windows' }}
+        run: ./scripts/buildRuntimeRewatch.sh
+        shell: bash
+
+      - name: Build runtime/stdlib with bsb (Windows)
+        if: ${{ runner.os == 'Windows' }}
         run: ./scripts/buildRuntime.sh
         shell: bash
 
@@ -440,8 +418,7 @@ jobs:
           path: lib/ocaml
 
   pkg-pr-new:
-    needs: 
-      - build-rewatch
+    needs:
       - build-compiler
     runs-on: ubuntu-24.04-arm
     steps:
@@ -457,7 +434,7 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: "@(binaries-*|rewatch-*|lib-ocaml)"
+          pattern: "@(binaries-*|lib-ocaml)"
 
       - name: Move artifacts into packages
         run: .github/workflows/moveArtifacts.sh
@@ -467,27 +444,23 @@ jobs:
         run: |
           yarn dlx pkg-pr-new publish "." "./packages/@rescript/*"
 
-  installationTest:
+  test-integration:
     needs:
       - pkg-pr-new
     strategy:
       fail-fast: false
       matrix:
         include:
           - os: macos-13
-            node-target: darwin-x64
           - os: macos-14
-            node-target: darwin-arm64
           - os: ubuntu-24.04
-            node-target: linux-x64
           - os: ubuntu-24.04-arm
-            node-target: linux-arm64
           - os: windows-latest
-            node-target: win32-x64
     runs-on: ${{ matrix.os }}
+    env:
+      RUST_BACKTRACE: "1"
     steps:
       - name: Checkout
-        id: checkout
         uses: actions/checkout@v4
 
       - name: Use Node.js
@@ -521,11 +494,22 @@ jobs:
         shell: bash
         working-directory: ${{ steps.tmp-dir.outputs.path }}
 
+      - name: Install ReScript package in rewatch/testrepo
+        if: runner.os == 'Linux'
+        run: |
+          COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
+          yarn add "rescript@https://pkg.pr.new/rescript-lang/rescript@${COMMIT_SHA::7}"
+        shell: bash
+        working-directory: rewatch/testrepo
+
+      - name: Run rewatch integration tests
+        # Currently failing on Windows and intermittently on macOS
+        if: runner.os == 'Linux'
+        run: make test-rewatch-ci
+
   publish:
     needs:
-      - build-rewatch
-      - build-compiler
-      - installationTest
+      - test-integration
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-24.04-arm
     steps:
@@ -542,7 +526,7 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: "@(binaries-*|rewatch-*|lib-ocaml)"
+          pattern: "@(binaries-*|lib-ocaml)"
 
       - name: Move artifacts into packages
         run: .github/workflows/moveArtifacts.sh

.github/workflows/moveArtifacts.sh

@@ -29,12 +29,12 @@ check_statically_linked() {
 mv lib-ocaml lib/ocaml
 
 # @rescript/{target}
-chmod +x binaries-*/*.exe rewatch-*/*.exe
-mv -f binaries-darwin-x64/* rewatch-darwin-x64/* "packages/@rescript/darwin-x64/bin"
-mv -f binaries-darwin-arm64/* rewatch-darwin-arm64/* "packages/@rescript/darwin-arm64/bin"
-mv -f binaries-linux-x64/* rewatch-linux-x64/* "packages/@rescript/linux-x64/bin"
-mv -f binaries-linux-arm64/* rewatch-linux-arm64/* "packages/@rescript/linux-arm64/bin"
-mv -f binaries-win32-x64/* rewatch-win32-x64/* "packages/@rescript/win32-x64/bin"
+chmod +x binaries-*/*.exe
+mv -f binaries-darwin-x64/* "packages/@rescript/darwin-x64/bin"
+mv -f binaries-darwin-arm64/* "packages/@rescript/darwin-arm64/bin"
+mv -f binaries-linux-x64/* "packages/@rescript/linux-x64/bin"
+mv -f binaries-linux-arm64/* "packages/@rescript/linux-arm64/bin"
+mv -f binaries-win32-x64/* "packages/@rescript/win32-x64/bin"
 check_statically_linked "packages/@rescript/linux-x64/bin"
 check_statically_linked "packages/@rescript/linux-arm64/bin"
 

CHANGELOG.md

@@ -44,6 +44,7 @@
 
 - AST: Add bar location to `case`. https://github.com/rescript-lang/rescript/pull/7407
 - Clean up lazy from ASTs and back-end. https://github.com/rescript-lang/rescript/pull/7474
+- Compile runtime with rewatch and add rewatch tests to the compiler repo. https://github.com/rescript-lang/rescript/pull/7422/files
 
 #### :nail_care: Polish
 

Makefile

@@ -16,8 +16,8 @@ dce:
 	reanalyze.exe -dce-cmt _build/default/compiler
 
 rewatch:
-	cargo build --manifest-path rewatch/Cargo.toml
-	cp rewatch/target/debug/rewatch rewatch
+	cargo build --manifest-path rewatch/Cargo.toml --release
+	cp rewatch/target/release/rewatch rewatch
 	./scripts/copyExes.js --rewatch
 
 ninja/ninja:
@@ -46,15 +46,25 @@ test-syntax-roundtrip:
 test-gentype:
 	make -C tests/gentype_tests/typescript-react-example clean test
 
-test-all: test test-gentype test-analysis test-tools
+test-rewatch:
+	bash ./rewatch/tests/suite-ci.sh
+
+test-rewatch-ci:
+	bash ./rewatch/tests/suite-ci.sh node_modules/.bin/rewatch
+
+test-all: test test-gentype test-analysis test-tools test-rewatch
 
 reanalyze:
 	reanalyze.exe -set-exit-code -all-cmt _build/default/compiler _build/default/tests -exclude-paths compiler/outcome_printer,compiler/ml,compiler/frontend,compiler/ext,compiler/depends,compiler/core,compiler/common,compiler/cmij,compiler/bsb_helper,compiler/bsb
 
-lib:
+lib-bsb:
 	./scripts/buildRuntime.sh
 	./scripts/prebuilt.js
 
+lib:
+	./scripts/buildRuntimeRewatch.sh
+	./scripts/prebuilt.js
+
 artifacts: lib
 	./scripts/npmPack.js --updateArtifactList
 

cli/rewatch.js

@@ -3,8 +3,10 @@
 // @ts-check
 
 import * as child_process from "node:child_process";
-import { rewatch_exe } from "./common/bins.js";
+import { rewatch_exe, bsc_exe } from "./common/bins.js";
 
 const args = process.argv.slice(2);
 
-child_process.spawnSync(rewatch_exe, args, { stdio: "inherit" });
+child_process.spawnSync(rewatch_exe, [...args, "--bsc-path", bsc_exe], {
+  stdio: "inherit",
+});

rewatch/.cargo/config.toml

@@ -0,0 +1,4 @@
+[alias]
+build-m1-release = "RUSTFLAGS=\"-C target-cpu=apple-m1\" cargo build --release"
+build-docs = "cargo doc --no-deps --document-private-items --target-dir ./docs"
+

rewatch/.gitignore

@@ -0,0 +1,3 @@
+/target
+.DS_Store
+/docs