Merge branch 'main' into Add-llvm-21-support

Author: mcbarton

.github/workflows/emscripten.yml

@@ -842,7 +842,13 @@ jobs:
         .\emsdk\emsdk_env.ps1
         $env:PWD_DIR= $PWD.Path
         $env:SYSROOT_PATH="$env:EMSDK/upstream/emscripten/cache/sysroot"
-        $env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+        echo "SYSROOT_PATH=$env:SYSROOT_PATH"
+        echo "SYSROOT_PATH=$env:SYSROOT_PATH" >> $env:GITHUB_ENV
+        
+        $env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
+        echo "PREFIX=$env:PREFIX"
+        echo "PREFIX=$env:PREFIX" >> $env:GITHUB_ENV
+        
         $env:CMAKE_PREFIX_PATH=$env:PREFIX
         $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
         
@@ -945,6 +951,11 @@ jobs:
         Error-OnFailure { emrun.bat --browser="chrome.exe" --kill_exit --timeout 60 --browser-args="--headless --no-sandbox"  DynamicLibraryManagerTests.html }
         cd ..\..
         Error-OnFailure { emmake make -j ${{ env.ncpus }} install }
+        echo "SYSROOT_PATH=$env:SYSROOT_PATH" >> $GITHUB_ENV
+        echo "CPPINTEROP_DIR=$env:CPPINTEROP_DIR" >> $GITHUB_ENV
+        echo "LLVM_BUILD_DIR=$env:LLVM_BUILD_DIR" >> $GITHUB_ENV
+        echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+        echo "PREFIX=$env:PREFIX" >> $GITHUB_ENV
 
     - name: Build and Test/Install CppInterOp on Windows systems (static library)
       if: ${{ runner.os == 'windows' }}
@@ -1047,7 +1058,7 @@ jobs:
         echo "Running DynamicLibraryManagerTests in Chromium"
         Error-OnFailure{ emrun.bat --browser="chrome.exe" --kill_exit --timeout 60 --browser-args="--headless --no-sandbox"  DynamicLibraryManagerTests.html }
 
-    - name: Build xeus-cpp
+    - name: Build xeus-cpp on Unix Systems
       if: ${{ runner.os != 'windows' }}
       shell: bash -l {0}
       run: |
@@ -1071,7 +1082,31 @@ jobs:
           ..
         emmake make -j ${{ env.ncpus }} install
 
-    - name: Test xeus-cpp C++ Emscripten
+    - name: Build xeus-cpp on Windows systems
+      if: ${{ runner.os == 'windows' }}
+      shell: powershell
+      run: |
+        .\emsdk\emsdk activate ${{matrix.emsdk_ver}}
+        .\emsdk\emsdk_env.ps1
+        micromamba activate CppInterOp-wasm
+        git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+        cd .\xeus-cpp
+        mkdir build
+        pushd build
+        $env:CMAKE_PREFIX_PATH="${{ env.PREFIX }}"
+        $env:CMAKE_SYSTEM_PREFIX_PATH="${{ env.PREFIX }}"
+        emcmake cmake `
+          -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}          `
+          -DCMAKE_PREFIX_PATH=${{ env.PREFIX }}             `
+          -DCMAKE_INSTALL_PREFIX=${{ env.PREFIX }}          `
+          -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON               `
+          -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON            `
+          -DXEUS_CPP_RESOURCE_DIR=${{ env.LLVM_BUILD_DIR }}\lib\clang\${{ matrix.clang-runtime }} `
+          -DSYSROOT_PATH=${{ env.SYSROOT_PATH }}                `
+          ..
+        emmake make -j ${{ env.ncpus }} install
+
+    - name: Test xeus-cpp C++ Emscripten on Unix Systems
       if: ${{ runner.os != 'windows' }}
       shell: bash -l {0}
       run: |
@@ -1080,6 +1115,30 @@ jobs:
         cd ./xeus-cpp/build/test
         node test_xeus_cpp.js
 
+    - name: Test xeus-cpp C++ Emscripten on Windows Systems
+      if: ${{ runner.os == 'windows' }}
+      shell: powershell
+      run: |
+        function Error-OnFailure {
+          param (
+              [Parameter(Mandatory)]
+              [ScriptBlock]$Command
+          )
+
+         & $Command
+
+         if ($LASTEXITCODE -ne 0) {
+            exit $LASTEXITCODE
+         }
+        }
+        micromamba activate CppInterOp-wasm
+        cd .\xeus-cpp\build\test
+        node test_xeus_cpp.js
+        echo "Running test_xeus_cpp in Firefox"
+        Error-OnFailure { emrun.bat --browser="firefox.exe" --kill_exit --timeout 60 --browser-args="--headless"  test_xeus_cpp.html }
+        echo "Running test_xeus_cpp in Chromium"
+        Error-OnFailure { emrun.bat --browser="chrome.exe" --kill_exit --timeout 60 --browser-args="--headless --no-sandbox"  test_xeus_cpp.html }
+
     - name: Jupyter Lite integration
       if: ${{ runner.os != 'windows' }}
       shell: bash -l {0}
@@ -1088,3 +1147,12 @@ jobs:
           micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
           micromamba activate xeus-lite-host
           jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --contents notebooks/xeus-cpp-lite-demo.ipynb --contents notebooks/smallpt.ipynb --contents notebooks/images/marie.png --contents notebooks/audio/audio.wav --output-dir dist
+
+    - name: Jupyter Lite integration
+      if: ${{ runner.os == 'windows' }}
+      shell: powershell
+      run: |
+          cd .\xeus-cpp\
+          micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+          micromamba activate xeus-lite-host
+          jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --contents notebooks/xeus-cpp-lite-demo.ipynb --contents notebooks/smallpt.ipynb --contents notebooks/images/marie.png --contents notebooks/audio/audio.wav --output-dir dist

Emscripten-build-instructions.md

@@ -178,7 +178,7 @@ export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
 and
 
 ```powershell
-$env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+$env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
 $env:CMAKE_PREFIX_PATH=$env:PREFIX
 $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
 ```
@@ -337,7 +337,7 @@ emmake make -j $(nproc --all) install
 ## Xeus-cpp-lite Wasm Build Instructions
 
 A project which makes use of the wasm build of CppInterOp is xeus-cpp. xeus-cpp is a C++ Jupyter kernel. Assuming you are in  
-the CppInterOp build folder, you can build the wasm version of xeus-cpp by executing (replace LLVM_VERSION with the version
+the CppInterOp build folder, you can build the wasm version of xeus-cpp on Linux/MacOS by executing (replace LLVM_VERSION with the version
 of llvm you are building against)
 
 ```bash
@@ -359,7 +359,28 @@ emcmake cmake \
  emmake make -j $(nproc --all) install
 ```
 
-To build and test Jupyter Lite with this kernel locally you can execute the following
+and on Windows by executing
+
+```powershell
+cd ..\..
+git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+$env:LLVM_VERSION=20
+cd .\xeus-cpp
+mkdir build
+cd build
+emcmake cmake `
+          -DCMAKE_BUILD_TYPE=Release                                     `
+          -DCMAKE_PREFIX_PATH="$env:PREFIX"                                    `
+          -DCMAKE_INSTALL_PREFIX="$env:PREFIX"                                 `
+          -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON                            `
+          -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON                         `
+          -DXEUS_CPP_RESOURCE_DIR="$env:LLVM_BUILD_DIR/lib/clang/$env:LLVM_VERSION" `
+          -DSYSROOT_PATH="$env:SYSROOT_PATH"                                   `
+          ..
+ emmake make -j $(nproc --all) install
+```
+
+To build and test Jupyter Lite with this kernel locally on Linux/MacOS you can execute the following
 
 ```bash
 cd ../..
@@ -373,3 +394,18 @@ jupyter lite serve --XeusAddon.prefix=$PREFIX \
                    --XeusAddon.mounts="$PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" \
                    --XeusAddon.mounts="$PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
 ```
+
+and on Windows execute
+
+```powershell
+cd ..\..
+micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+micromamba activate xeus-lite-host
+jupyter lite serve --XeusAddon.prefix="$env:PREFIX" `
+                   --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb `
+                   --contents xeus-cpp/notebooks/smallpt.ipynb `
+                   --contents xeus-cpp/notebooks/images/marie.png `
+                   --contents xeus-cpp/notebooks/audio/audio.wav `
+                   --XeusAddon.mounts="$env:PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" `
+                   --XeusAddon.mounts="$env:PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
+```

docs/DevelopersDocumentation.rst

@@ -423,54 +423,12 @@ library files and run pytest:
    python -m pip install pytest
    python -m pytest -sv
 
-###################################
+***********************************
  CppInterOp Internal Documentation
-###################################
+***********************************
 
 CppInterOp maintains an internal Doxygen documentation of its components.
 Internal documentation aims to capture intrinsic details and overall usage of
 code components. The goal of internal documentation is to make the codebase
 easier to understand for the new developers. Internal documentation can be
 visited : `here <build/html/index.html>`_
-
-**************************************
- Multiple Interpreter & Thread-Safety
-**************************************
-
-CppInterOp allows the user to create multiple interpreters at a time and
-use those interpreters. The interpreters that are created are stored in a
-stack and a map. The stack is used to enable the model where the user
-wants to create a temporary interpreter and destroy it after performing a
-few operations. In such a use case, the top of the stack is the only
-interpreter in use at any given point in time.
-
-The map is used to store the mapping from :code:`clang::ASTContext` to
-:code:`Cpp::InterpreterInfo`. This is required to figure out which
-interpreter an object belongs to. Say the library user performs the
-following operations:
-
-1. Create an Interpreter
-2. Compile some code with variable :code:`a`
-3. Create another Interpreter
-4. Performs :code:`Cpp::GetVariableOffset(a)`
-
-In step 4, the top of the stack is an interpreter without the definition of
-:code:`a`. And we cannot use it to figure out the address of :code:`a`.
-The :code:`clang::Decl` passed to :code:`Cpp::GetVariableOffset` is used to
-retrieve the :code:`clang::ASTContext`, using
-:code:`clang::Decl::getASTContext`. We then use the map to figure out the
-exact Interpreter Instance this :code:`clang::Decl` belongs to and perform
-the operation.
-
-A shortcoming of this is that if the CppInterOp accepts a
-:code:`clang::QualType` instead of :code:`clang::Decl`, then it is not
-possible to get the :code:`clang::ASTContext` from the :code:`clang::QualType`.
-In such cases, we iterate over the Allocator of all the Interpreters in our
-stack and figure out which :code:`clang::ASTContext` allocated this
-:code:`clang::QualType`. This is a very expensive operation. But there is no
-alternative to this.
-
-For **thread-safety**, we introduce a lock for each of the interpreters we
-create. And lock only that one specific interpreter when required. We also
-have 2 global locks, one for LLVM, and another is used to lock operations
-performed on the interpreter stack and the map itself.

docs/Emscripten-build-instructions.rst

@@ -203,7 +203,7 @@ and
 
 .. code:: powershell
 
-   $env:PREFIX="%CONDA_PREFIX%/envs/CppInterOp-wasm"
+   $env:PREFIX="$env:MAMBA_ROOT_PREFIX/envs/CppInterOp-wasm"
    $env:CMAKE_PREFIX_PATH=$env:PREFIX
    $env:CMAKE_SYSTEM_PREFIX_PATH=$env:PREFIX
 
@@ -358,8 +358,8 @@ Assuming it passes all test you can install by executing the following.
 
 A project which makes use of the wasm build of CppInterOp is xeus-cpp.
 xeus-cpp is a C++ Jupyter kernel. Assuming you are in the CppInterOp
-build folder, you can build the wasm version of xeus-cpp by executing
-(replace LLVM_VERSION with the version of llvm you are building against)
+build folder, you can build the wasm version of xeus-cpp on Linux/MacOS 
+by executing (replace LLVM_VERSION with the version of llvm you are building against)
 
 .. code:: bash
 
@@ -380,7 +380,28 @@ build folder, you can build the wasm version of xeus-cpp by executing
            ..
    emmake make -j $(nproc --all) install
 
-To build and test Jupyter Lite with this kernel locally you can execute the following
+and on Windows by executing 
+
+.. code:: powershell
+
+   cd ..\..
+   git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+   $env:LLVM_VERSION=20
+   cd .\xeus-cpp
+   mkdir build
+   cd build
+   emcmake cmake `
+           -DCMAKE_BUILD_TYPE=Release                                     `
+           -DCMAKE_PREFIX_PATH="$env:PREFIX"                              `
+           -DCMAKE_INSTALL_PREFIX="$env:PREFIX"                           `
+           -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON                            `
+           -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON                         `
+	        -DXEUS_CPP_RESOURCE_DIR="$env:LLVM_BUILD_DIR/lib/clang/$env:LLVM_VERSION" `
+           -DSYSROOT_PATH="$env:SYSROOT_PATH"                              `
+           ..
+   emmake make -j $(nproc --all) install
+
+To build and test Jupyter Lite with this kernel locally on Linux/MacOS you can execute the following
 
 .. code:: bash
 
@@ -394,3 +415,18 @@ To build and test Jupyter Lite with this kernel locally you can execute the foll
                       --contents xeus-cpp/notebooks/audio/audio.wav \
                       --XeusAddon.mounts="$PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" \
                       --XeusAddon.mounts="$PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"
+
+and on Windows execute
+
+.. code:: powershell
+
+   cd ..\..
+   micromamba create -n xeus-lite-host jupyterlite-core=0.6 jupyterlite-xeus jupyter_server jupyterlab notebook python-libarchive-c -c conda-forge
+   micromamba activate xeus-lite-host
+   jupyter lite serve --XeusAddon.prefix="$env:PREFIX" `
+                      --contents xeus-cpp/notebooks/xeus-cpp-lite-demo.ipynb `
+                      --contents xeus-cpp/notebooks/smallpt.ipynb `
+                      --contents xeus-cpp/notebooks/images/marie.png ` 
+                      --contents xeus-cpp/notebooks/audio/audio.wav `
+                      --XeusAddon.mounts="$env:PREFIX/share/xeus-cpp/tagfiles:/share/xeus-cpp/tagfiles" `
+                      --XeusAddon.mounts="$env:PREFIX/etc/xeus-cpp/tags.d:/etc/xeus-cpp/tags.d"

environment-wasm.yml

@@ -1,9 +1,11 @@
 name: CppInterOp-wasm
 channels:
-  - https://repo.prefix.dev/emscripten-forge-dev
+  - https://prefix.dev/emscripten-forge-dev
+  - https://prefix.dev/conda-forge
 dependencies:
   - emscripten-abi==3.1.73
-  - nlohmann_json=3.12.0
+  - nlohmann_json
+  - nlohmann_json-abi
   - xeus-lite
   - xeus
   - cpp-argparse