Enable ThreadSanitizer across the entire multi-threaded JIT pipeline

.ci/jit-debug-test.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# JIT Debug Test Script
+# This script tests JIT compiler with debug mode enabled to catch issues early
+
+set -e
+
+PARALLEL="${PARALLEL:--j$(nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 4)}"
+
+echo "======================================"
+echo "JIT Debug Mode Test"
+echo "======================================"
+
+# Test 1: Standard JIT with debug
+echo ""
+echo "Test 1: Building with ENABLE_JIT_DEBUG=1..."
+make distclean
+make ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 $PARALLEL
+
+echo ""
+echo "Running basic tests with JIT debug..."
+make ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 check
+
+# Test 2: JIT with EXT_C=0 and debug (regression test)
+echo ""
+echo "Test 2: Building with ENABLE_EXT_C=0 ENABLE_JIT_DEBUG=1..."
+make distclean
+make ENABLE_EXT_C=0 ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 $PARALLEL
+
+echo ""
+echo "Running tests with EXT_C=0 and JIT debug..."
+make ENABLE_EXT_C=0 ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 check
+
+# Test 3: JIT with various extension combinations
+echo ""
+echo "Test 3: Testing multiple JIT configurations with debug..."
+for config in \
+    "ENABLE_EXT_A=0" \
+    "ENABLE_EXT_F=0" \
+    "ENABLE_EXT_M=0" \
+    "ENABLE_Zba=0" \
+    "ENABLE_Zbb=0"; do
+    echo ""
+    echo "Testing: $config with JIT debug"
+    make distclean
+    make $config ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 $PARALLEL
+    make $config ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 check
+done
+
+echo ""
+echo "======================================"
+echo "All JIT debug tests passed!"
+echo "======================================"

.github/workflows/main.yml

@@ -135,14 +135,15 @@ jobs:
     - name: default build using emcc
       if: success()
       run: |
+            make distclean
             make CC=emcc ENABLE_JIT=0 $PARALLEL
 
     - name: default build for system emulation using emcc
       if: success()
       run: |
             make distclean
             make CC=emcc ENABLE_SYSTEM=1 ENABLE_JIT=0 $PARALLEL
-            make distclean ENABLE_SYSTEM=1
+            make distclean
 
     - name: Build with various optimization levels
       if: success()
@@ -301,17 +302,56 @@ jobs:
         githubToken: ${{ github.token }}
         # No 'sudo' is available
         install: |
-          apt update -qq
-          apt install -yqq make git curl wget clang libsdl2-dev libsdl2-mixer-dev lsb-release software-properties-common gnupg bc
-          which wget || echo "WARNING: wget not found after installation"
+          # Retry apt update with exponential backoff for mirror sync issues
+          # Note: dep11 (AppStream metadata) failures are non-critical for build tools
+          set -o pipefail
+          for i in 1 2 3; do
+            if apt update -qq --allow-releaseinfo-change 2>&1 | tee /tmp/apt-update.log; then
+              APT_EXIT=0
+            else
+              APT_EXIT=$?
+            fi
+            # Check for critical failures (package indices), ignore dep11 metadata
+            # Include InRelease which is the combined Release+Release.gpg file
+            if [ $APT_EXIT -eq 0 ] && ! grep -E "Failed to fetch.*/(Packages|Sources|Release|InRelease)" /tmp/apt-update.log; then
+              echo "apt update succeeded (core package lists available)"
+              break
+            fi
+            if [ $i -lt 3 ]; then
+              delay=$((i * 30))
+              echo "apt update attempt $i: errors detected (exit=$APT_EXIT), waiting ${delay}s..."
+              sleep $delay
+            else
+              echo "Warning: Proceeding after 3 attempts - some package lists may be incomplete"
+            fi
+          done
+          # Install packages - exit 0 even if dep11 metadata is incomplete
+          apt install -yqq make git curl wget clang libsdl2-dev libsdl2-mixer-dev lsb-release software-properties-common gnupg bc 2>&1 | tee /tmp/apt-install.log || true
+          # Verify critical packages were installed
+          for pkg in make git curl clang bc; do
+            if ! command -v $pkg >/dev/null 2>&1; then
+              echo "ERROR: Critical package $pkg failed to install!"
+              cat /tmp/apt-install.log
+              exit 1
+            fi
+          done
+          echo "All critical build tools installed successfully"
         # FIXME: gcc build fails on Aarch64/Linux hosts
         env: |
           CC: clang-18
         # Append custom commands here
         run: |
           # Verify and install wget if needed (workaround for install step issues)
           if ! command -v wget > /dev/null 2>&1; then
-            apt update -qq && apt install -yqq wget
+            echo "wget not found, attempting to install..."
+            apt update -qq --allow-releaseinfo-change 2>&1 | tee /tmp/apt-update-wget.log || true
+            apt install -yqq wget 2>&1 | tee /tmp/wget-install.log || true
+            if ! command -v wget > /dev/null 2>&1; then
+              echo "ERROR: wget installation failed!"
+              cat /tmp/wget-install.log
+              exit 1
+            fi
+            echo "wget installed successfully"
           fi
           git config --global --add safe.directory ${{ github.workspace }}
           git config --global --add safe.directory ${{ github.workspace }}/src/softfloat
@@ -435,14 +475,15 @@ jobs:
      - name: default build using emcc
        if: success()
        run: |
+             make distclean
              make CC=emcc ENABLE_JIT=0 $PARALLEL
 
      - name: default build for system emulation using emcc
        if: success()
        run: |
              make distclean
              make CC=emcc ENABLE_SYSTEM=1 ENABLE_JIT=0 $PARALLEL
-             make distclean ENABLE_SYSTEM=1
+             make distclean
 
      - name: check + tests
        if: success()
@@ -499,6 +540,15 @@ jobs:
                fi
              done
 
+     - name: JIT debug test
+       env:
+         CC: ${{ steps.install_cc.outputs.cc }}
+       run: |
+             # Run JIT tests with debug mode to catch register allocation and cache coherency issues
+             make distclean && make ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 check $PARALLEL
+             make distclean && make ENABLE_EXT_C=0 ENABLE_JIT=1 ENABLE_JIT_DEBUG=1 check $PARALLEL
+       if: ${{ always() }}
+
      - name: undefined behavior test
        if: (success() || failure()) && steps.install_cc.outputs.cc == 'clang'  # gcc on macOS/arm64 does not support sanitizers
        env:

Makefile

@@ -20,13 +20,6 @@ CFLAGS += -include src/common.h -Isrc/
 
 OBJS_EXT :=
 
-# In the system test suite, the executable is an ELF file (e.g., MMU).
-# However, the Linux kernel emulation includes the Image, DT, and
-# root filesystem (rootfs). Therefore, the test suite needs this
-# flag to load the ELF and differentiate it from the kernel emulation.
-ENABLE_ELF_LOADER ?= 0
-$(call set-feature, ELF_LOADER)
-
 # Enable MOP fusion, easier for ablation study
 ENABLE_MOP_FUSION ?= 1
 $(call set-feature, MOP_FUSION)
@@ -80,6 +73,49 @@ endif
 ENABLE_ARCH_TEST ?= 0
 $(call set-feature, ARCH_TEST)
 
+# In the system test suite, the executable is an ELF file (e.g., MMU).
+# However, the Linux kernel emulation includes the Image, DT, and
+# root filesystem (rootfs). Therefore, the test suite needs this
+# flag to load the ELF and differentiate it from the kernel emulation.
+# User-space emulation (SYSTEM=0) always needs ELF loader, except for architecture tests.
+ifeq ($(ENABLE_SYSTEM), 0)
+    ifneq ($(ENABLE_ARCH_TEST), 1)
+        override ENABLE_ELF_LOADER := 1
+    else
+        ENABLE_ELF_LOADER ?= 0
+    endif
+else
+    ENABLE_ELF_LOADER ?= 0
+endif
+$(call set-feature, ELF_LOADER)
+
+# ThreadSanitizer support
+# TSAN on x86-64 memory layout:
+#   Shadow: 0x02a000000000 - 0x7cefffffffff (reserved by TSAN)
+#   App:    0x7cf000000000 - 0x7ffffffff000 (usable by application)
+#
+# We use MAP_FIXED to allocate FULL4G's 4GB memory at a fixed address
+# (0x7d0000000000) within TSAN's app range, ensuring compatibility.
+#
+# IMPORTANT: TSAN requires ASLR (Address Space Layout Randomization) to be
+# disabled to prevent system allocations from landing in TSAN's shadow memory.
+# Tests are run with 'setarch $(uname -m) -R' to disable ASLR.
+ENABLE_TSAN ?= 0
+ifeq ("$(ENABLE_TSAN)", "1")
+override ENABLE_SDL := 0       # SDL (uninstrumented system lib) creates threads TSAN cannot track
+override ENABLE_LTO := 0       # LTO interferes with TSAN instrumentation
+CFLAGS += -DTSAN_ENABLED       # Signal code to use TSAN-compatible allocations
+# Disable ASLR for TSAN tests to prevent allocations in TSAN shadow memory
+# Note: setarch is Linux-only; macOS requires different approach (SIP disable)
+ifeq ($(UNAME_S),Linux)
+BIN_WRAPPER = setarch $(shell uname -m) -R
+else
+BIN_WRAPPER =
+endif
+else
+BIN_WRAPPER =
+endif
+
 # Enable link-time optimization (LTO)
 ENABLE_LTO ?= 1
 ifeq ($(call has, LTO), 1)
@@ -280,6 +316,11 @@ ENABLE_JIT ?= 0
 $(call set-feature, JIT)
 ifeq ($(call has, JIT), 1)
     OBJS_EXT += jit.o
+    # JIT debug mode for early issue detection in CI/CD
+    ENABLE_JIT_DEBUG ?= 0
+    ifeq ("$(ENABLE_JIT_DEBUG)", "1")
+        CFLAGS += -DENABLE_JIT_DEBUG=1
+    endif
     ENABLE_T2C ?= 1
     $(call set-feature, T2C)
     ifeq ($(call has, T2C), 1)
@@ -332,6 +373,12 @@ CFLAGS += -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover=all
 LDFLAGS += -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover=all
 endif
 
+# ThreadSanitizer flags (ENABLE_TSAN is set earlier to override SDL/FULL4G)
+ifeq ("$(ENABLE_TSAN)", "1")
+CFLAGS += -fsanitize=thread -g
+LDFLAGS += -fsanitize=thread
+endif
+
 $(OUT)/emulate.o: CFLAGS += -foptimize-sibling-calls -fomit-frame-pointer -fno-stack-check -fno-stack-protector
 
 # .DEFAULT_GOAL should be set to all since the very first target is not all
@@ -350,7 +397,7 @@ DTB_DEPS := $(BUILD_DTB) $(BUILD_DTB2C)
 endif
 endif
 
-all: config $(DTB_DEPS) $(BUILD_DTB) $(BUILD_DTB2C) $(BIN)
+all: config $(DTB_DEPS) $(BIN)
 
 OBJS := \
     map.o \
@@ -395,7 +442,7 @@ $(OUT):
 
 $(BIN): $(OBJS) $(DEV_OBJS) | $(OUT)
 	$(VECHO) "  LD\t$@\n"
-	$(Q)$(CC) -o $@ $(CFLAGS_emcc) $^ $(LDFLAGS)
+	$(Q)$(CC) -o $@ $(CFLAGS_emcc) $(filter-out %.dtb %.h,$^) $(LDFLAGS)
 
 $(CONFIG_FILE): FORCE
 	$(Q)mkdir -p $(OUT)
@@ -445,7 +492,7 @@ define check-test
 $(Q)true; \
 $(PRINTF) "Running $(3) ... "; \
 OUTPUT_FILE="$$(mktemp)"; \
-if (LC_ALL=C $(BIN) $(1) $(2) > "$$OUTPUT_FILE") && \
+if (LC_ALL=C $(BIN_WRAPPER) $(BIN) $(1) $(2) > "$$OUTPUT_FILE") && \
    [ "$$(cat "$$OUTPUT_FILE" | $(LOG_FILTER) | $(4))" = "$(5)" ]; then \
     $(call notice, [OK]); \
 else \

mk/toolchain.mk

@@ -22,10 +22,10 @@ ifneq ($(shell $(CC) --version | head -n 1 | grep emcc),)
                 $(warning $(SDL_MUSIC_CANNOT_PLAY_WARNING))
             endif
         else
-	    $(warning $(SDL_MUSIC_CANNOT_PLAY_WARNING))
+        $(warning $(SDL_MUSIC_CANNOT_PLAY_WARNING))
         endif
     else
-	$(warning $(SDL_MUSIC_CANNOT_PLAY_WARNING))
+        $(warning $(SDL_MUSIC_CANNOT_PLAY_WARNING))
     endif
 
     # see commit 165c1a3 of emscripten

mk/wasm.mk

@@ -166,3 +166,21 @@ start-web: $(start_web_deps)
 .PHONY: check-demo-dir-exist start-web
 
 endif
+
+# For SYSTEM mode, DTB needs to be built regardless of whether we're using emcc
+# DTB is only built when SYSTEM=1 and ELF_LOADER=0
+ifeq ($(call has, SYSTEM), 1)
+ifeq ($(call has, ELF_LOADER), 0)
+# Add DTB as dependency for compilation stages
+# This is used by mk/system.mk for device object files
+deps_emcc += $(BUILD_DTB) $(BUILD_DTB2C)
+
+# For emcc builds: ensure DTB exists before emcc embeds it
+# Make BIN directly depend on DTB files as regular prerequisites
+# This will cause them to be built, but they'll also be passed to the linker
+# We need to filter them out in the linker command
+ifeq ("$(CC_IS_EMCC)", "1")
+$(BIN): $(BUILD_DTB) $(BUILD_DTB2C)
+endif
+endif
+endif