Add WebAssembly build, cockle and terminal deployments, and testing

.gitignore

@@ -2,3 +2,4 @@ build/
 __pycache__/
 .cache/
 compile_commands.json
+serve.log

README.md

@@ -25,3 +25,13 @@ The CLI is tested using `python`. From the top-level directory:
 ```bash
 pytest -v
 ```
+
+# WebAssembly build and deployment
+
+The `wasm` directory contains everything needed to build the local `git2cpp` source code as an
+WebAssembly [Emscripten-forge](https://emscripten-forge.org/) package, create local
+[cockle](https://github.com/jupyterlite/cockle) and
+[JupyterLite terminal](https://github.com/jupyterlite/terminal) deployments that run in a browser,
+and test the WebAssembly build.
+
+See the `README.md` in the `wasm` directory for further details.

test/conftest.py

@@ -3,6 +3,10 @@
 import pytest
 import subprocess
 
+GIT2CPP_TEST_WASM = os.getenv('GIT2CPP_TEST_WASM') == "1"
+
+if GIT2CPP_TEST_WASM:
+    from .conftest_wasm import *
 
 # Fixture to run test in current tmp_path
 @pytest.fixture
@@ -14,7 +18,10 @@ def run_in_tmp_path(tmp_path):
 
 @pytest.fixture(scope='session')
 def git2cpp_path():
-    return Path(__file__).parent.parent / 'build' / 'git2cpp'
+    if GIT2CPP_TEST_WASM:
+        return 'git2cpp'
+    else:
+        return Path(__file__).parent.parent / 'build' / 'git2cpp'
 
 @pytest.fixture
 def xtl_clone(git2cpp_path, tmp_path, run_in_tmp_path):

test/conftest_wasm.py

@@ -0,0 +1,55 @@
+# Extra fixtures used for wasm testing.
+from functools import partial
+from pathlib import Path
+from playwright.sync_api import Page
+import pytest
+import subprocess
+import time
+
+@pytest.fixture(scope="session", autouse=True)
+def run_web_server():
+    with open('serve.log', 'w') as f:
+        cwd = Path(__file__).parent.parent / 'wasm/test'
+        proc = subprocess.Popen(
+            ['npm', 'run', 'serve'], stdout=f, stderr=f, cwd=cwd
+        )
+        # Wait a bit until server ready to receive connections.
+        time.sleep(0.3)
+        yield
+        proc.terminate()
+
+@pytest.fixture(scope="function", autouse=True)
+def load_page(page: Page):
+    # Load web page at start of every test.
+    page.goto("http://localhost:8000")
+    page.locator("#loaded").wait_for()
+
+def subprocess_run(
+    page: Page,
+    cmd: list[str],
+    *,
+    capture_output: bool = False,
+    cwd: str | None = None,
+    text: bool | None = None
+) -> subprocess.CompletedProcess:
+    if cwd is not None:
+        raise RuntimeError('cwd is not yet supported')
+
+    proc = page.evaluate("async cmd => window.cockle.shellRun(cmd)", cmd)
+    # TypeScript object is auto converted to Python dict.
+    # Want to return subprocess.CompletedProcess, consider namedtuple if this fails in future.
+    stdout = proc['stdout'] if capture_output else ''
+    stderr = proc['stderr'] if capture_output else ''
+    if not text:
+        stdout = stdout.encode("utf-8")
+        stderr = stderr.encode("utf-8")
+    return subprocess.CompletedProcess(
+        args=cmd,
+        returncode=proc['returncode'],
+        stdout=stdout,
+        stderr=stderr
+    )
+
+@pytest.fixture(scope="function", autouse=True)
+def mock_subprocess_run(page: Page, monkeypatch):
+    monkeypatch.setattr(subprocess, "run", partial(subprocess_run, page))

wasm/.gitignore

@@ -0,0 +1,2 @@
+serve/cockle/
+serve/lite/

wasm/Makefile

@@ -0,0 +1,28 @@
+default: build
+.PHONY: build clean rebuild serve test
+
+build:
+	make -C build $@
+	make -C cockle-deploy $@
+	make -C lite-deploy $@
+	make -C test $@
+
+# Rebuild after change in C++ source code.
+rebuild:
+	make -C build $@
+	make -C cockle-deploy $@
+	make -C lite-deploy $@
+	make -C test $@
+
+# Serve both cockle and JupyterLite deployments.
+serve:
+	npx static-handler serve
+
+test:
+	make -C test $@
+
+clean:
+	make -C build $@
+	make -C cockle-deploy $@
+	make -C lite-deploy $@
+	make -C test $@

wasm/README.md

@@ -0,0 +1,70 @@
+# Building and testing git2cpp in WebAssembly
+
+This directory contains everything needed to build the local `git2cpp` source code as an
+WebAssembly [Emscripten-forge](https://emscripten-forge.org/) package, create local
+[cockle](https://github.com/jupyterlite/cockle) and
+[JupyterLite terminal](https://github.com/jupyterlite/terminal) deployments that run in a browser,
+and test the WebAssembly build.
+
+It works on Linux and macOS but not Windows.
+
+There are 5 sub-directories:
+
+- `build`: build local `git2cpp` source code into an Emscripten-forge package.
+- `cockle-deploy`: create a `cockle` deployment in the `serve` directory.
+- `lite-deploy`: create a JupyterLite `terminal` deployment in the `serve` directory.
+- `serve`: where the two deployments are served from.
+- `test`: test the WebAssembly build.
+
+## Build and deploy
+
+The build, deploy and test process uses a separate `micromamba` environment defined in
+`wasm-environment.yml`. To set this up use from within this directory:
+
+```bash
+micromamba create -f wasm-environment.yml
+micromamba activate git2cpp-wasm
+```
+
+Then to build the WebAssembly package, both deployments and the testing resources use:
+
+```bash
+make
+```
+
+The local deployments in the `serve` directory can be manually checked using:
+
+```bash
+make serve
+```
+
+and open a web browser at http://localhost:8080/. Confirm that the local build of `git2cpp` is being
+used in the two deployments by running `cockle-config package` at the command line, the output
+should be something like:
+
+<img alt="cockle-config output" src="cockle-config.png">
+
+Note that the `source` for the `git2cpp` package is the local filesystem rather than from
+`prefix.dev`. The version number of `git2cpp` in this table is not necessarily correct as it is the
+version number of the current Emscripten-forge recipe rather than the version of the local `git2cpp`
+source code which can be checked using `git2cpp -v` at the `cockle`/`terminal` command line.
+
+## Test
+
+To test the WebAssembly build use:
+
+```bash
+make test
+```
+
+This runs (some of) the tests in the top-level `test` directory with various monkey patching so that
+`git2cpp` commands are executed in the browser.
+
+## Rebuild
+
+After making changes to the local `git2cpp` source code you can rebuild the WebAssembly package,
+both deployments and test code using:
+
+```bash
+make rebuild
+```

wasm/build/.gitignore

@@ -0,0 +1 @@
+em-forge-recipes/

wasm/build/Makefile

@@ -0,0 +1,31 @@
+default: build
+.PHONY: build clean clean-output-dir modify-recipe rebuild
+
+include ../common.mk
+
+EM_FORGE_RECIPES_REPO = https://github.com/emscripten-forge/recipes
+GIT2CPP_RECIPE_DIR = recipes/recipes_emscripten/git2cpp
+RATTLER_ARGS = --package-format tar-bz2 --target-platform emscripten-wasm32 -c https://repo.prefix.dev/emscripten-forge-dev -c microsoft -c conda-forge
+
+# Only want the git2cpp recipe from emscripten-forge/recipes repo, not the whole repo.
+# Note removing the .git directory otherwise `git clean -fxd` will not remove the directory.
+$(EM_FORGE_RECIPES_DIR):
+	git clone --no-checkout ${EM_FORGE_RECIPES_REPO} --depth 1 $@
+	cd $@ && git sparse-checkout init
+	cd $@ && git sparse-checkout set $(GIT2CPP_RECIPE_DIR)
+	cd $@ && git checkout main
+	rm -rf $@/.git
+
+modify-recipe: $(EM_FORGE_RECIPES_DIR)
+	python modify-recipe.py $(EM_FORGE_RECIPES_DIR)/$(GIT2CPP_RECIPE_DIR)
+
+build: modify-recipe
+	cd $(EM_FORGE_RECIPES_DIR) && rattler-build build $(RATTLER_ARGS) --recipe $(GIT2CPP_RECIPE_DIR)
+
+rebuild: clean-output-dir build
+
+clean:
+	rm -rf $(EM_FORGE_RECIPES_DIR)
+
+clean-output-dir:
+	rm -rf $(BUILT_PACKAGE_DIR)

wasm/build/modify-recipe.py

@@ -0,0 +1,49 @@
+# Modify the git2cpp emscripten-forge recipe to build from the local repo.
+# This can be called repeatedly and will produce the same output.
+
+from pathlib import Path
+import shutil
+import sys
+import yaml
+
+
+def quit(msg):
+    print(msg)
+    exit(1)
+
+if len(sys.argv) < 2:
+    quit(f'Usage: {sys.argv[0]} <recipe directory containing yaml file to modify>')
+
+input_dir = Path(sys.argv[1])
+if not input_dir.is_dir():
+    quit(f'{input_dir} should exist and be a directory')
+
+input_filename = input_dir / 'recipe.yaml'
+if not input_filename.is_file():
+    quit(f'{input_filename} should exist and be a file')
+
+# If backup does not exist create it.
+input_backup = input_dir / 'recipe_original.yaml'
+backup_exists = input_backup.exists()
+if not backup_exists:
+    shutil.copy(input_filename, input_backup)
+
+# Read and parse input backup file which is the original recipe file.
+with open(input_backup) as f:
+    recipe = yaml.safe_load(f)
+
+build_number = recipe['build']['number']
+print(f'  Changing build number from {build_number} to {build_number+1}')
+recipe['build']['number'] = build_number+1
+
+source = recipe['source']
+if not ('sha256' in source and 'url' in source):
+    raise RuntimeError('Expected recipe to have both a source sha256 and url')
+del source['sha256']
+del source['url']
+print('  Changing source to point to local git2cpp repo')
+source['path'] = '../../../../../../'
+
+# Overwrite recipe file.
+with open(input_filename, 'w') as f:
+    yaml.dump(recipe, f)

wasm/cockle-deploy/.gitignore

@@ -0,0 +1,3 @@
+cockle_wasm_env/
+node_modules/
+package-lock.json

wasm/cockle-deploy/Makefile

@@ -0,0 +1,16 @@
+default: build
+.PHONY: build clean clean-env rebuild
+
+include ../common.mk
+
+build:
+	npm install
+	COCKLE_WASM_EXTRA_CHANNEL=$(BUILT_PACKAGE_DIR) npm run build
+
+rebuild: clean-env build
+
+clean: clean-env
+	rm -rf ../serve/cockle node_modules/
+
+clean-env:
+	rm -rf cockle_wasm_env/

wasm/cockle-deploy/assets/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>cockle-deployment for git2cpp</title>
+    <script src="bundle.js"></script>
+</head>
+<body>
+    <div id="targetdiv"></div>
+</body>
+</html>

wasm/cockle-deploy/cockle-config-in.json

@@ -0,0 +1,19 @@
+{
+  "packages": {
+    "git2cpp": {},
+    "nano": {},
+    "tree": {},
+    "vim": {}
+  },
+  "aliases": {
+    "git": "git2cpp",
+    "vi": "vim"
+  },
+  "environment": {
+    "GIT_CORS_PROXY": "https://corsproxy.io/?url=",
+    "GIT_AUTHOR_NAME": "Jane Doe",
+    "GIT_AUTHOR_EMAIL": "jane.doe@blabla.com",
+    "GIT_COMMITTER_NAME": "Jane Doe",
+    "GIT_COMMITTER_EMAIL": "jane.doe@blabla.com"
+  }
+}

wasm/cockle-deploy/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "cockle-deploy",
+  "scripts": {
+    "build": "rspack build",
+    "postbuild": "npm run postbuild:wasm && npm run postbuild:index",
+    "postbuild:wasm": "node node_modules/@jupyterlite/cockle/lib/tools/prepare_wasm.js --copy ../serve/cockle/",
+    "postbuild:index": "cp assets/index.html ../serve/cockle/"
+  },
+  "main": "../serve/cockle/index.js",
+  "types": "../serve/cockle/index.d.ts",
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@rspack/cli": "^1.0.4",
+    "@rspack/core": "^1.0.4",
+    "css-loader": "^7.1.2",
+    "style-loader": "^4.0.0",
+    "ts-loader": "^9.5.1",
+    "typescript": "^5.4.5"
+  },
+  "dependencies": {
+    "@jupyterlite/cockle": "^1.2.0",
+    "@xterm/addon-fit": "^0.10.0",
+    "@xterm/xterm": "^5.5.0",
+    "deepmerge-ts": "^7.1.4"
+  }
+}

wasm/cockle-deploy/rspack.config.js

@@ -0,0 +1,29 @@
+const path = require('path');
+
+module.exports = {
+  mode: 'development',
+  entry: './src/index.ts',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+      {
+        test: /\.css$/,
+        use: [
+          'style-loader',
+          'css-loader'
+        ]
+      }
+    ]
+  },
+  resolve: {
+    extensions: ['.tsx', '.ts', '.js'],
+  },
+  output: {
+    filename: 'bundle.js',
+    path: path.resolve(__dirname, '../serve/cockle'),
+  }
+};