Merge remote-tracking branch 'upstream/master'

Author: miried

.github/workflows/ci.yml

@@ -13,35 +13,94 @@ defaults:
     shell: bash
 
 jobs:
-  build:
+  test:
+    name: test (${{ matrix.host_target }})
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-latest
-            host_target: x86_64-unknown-linux-gnu
-          - os: macos-14
-            host_target: aarch64-apple-darwin
-          - os: windows-latest
-            host_target: i686-pc-windows-msvc
+          - host_target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+          - host_target: i686-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: i386
+            gcc_cross: i686-linux-gnu
+          - host_target: aarch64-unknown-linux-gnu
+            os: ubuntu-24.04-arm
+          - host_target: armv7-unknown-linux-gnueabihf
+            os: ubuntu-24.04-arm
+            multiarch: armhf
+            gcc_cross: arm-linux-gnueabihf
+          - host_target: riscv64gc-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: riscv64
+            gcc_cross: riscv64-linux-gnu
+            qemu: true
+          - host_target: s390x-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: s390x
+            gcc_cross: s390x-linux-gnu
+            qemu: true
+          - host_target: aarch64-apple-darwin
+            os: macos-latest
+          - host_target: i686-pc-windows-msvc
+            os: windows-latest
+          - host_target: aarch64-pc-windows-msvc
+            os: windows-11-arm
     runs-on: ${{ matrix.os }}
     env:
       HOST_TARGET: ${{ matrix.host_target }}
     steps:
       - uses: actions/checkout@v4
+      - name: apt update
+        if: ${{ startsWith(matrix.os, 'ubuntu') }}
+        # The runners seem to have outdated apt repos sometimes
+        run: sudo apt update
+      - name: install qemu
+        if: ${{ matrix.qemu }}
+        run: sudo apt install qemu-user qemu-user-binfmt
+      - name: install multiarch
+        if: ${{ matrix.multiarch != '' }}
+        run: |
+          # s390x, ppc64el need Ubuntu Ports to be in the mirror list
+          sudo bash -c "echo 'https://ports.ubuntu.com/	priority:4' >> /etc/apt/apt-mirrors.txt"
+          # Add architecture
+          sudo dpkg --add-architecture ${{ matrix.multiarch }}
+          sudo apt update
+          # Install needed packages
+          sudo apt install $(echo "libatomic1: zlib1g-dev:" | sed 's/:/:${{ matrix.multiarch }}/g')
+      - name: Install rustup on Windows ARM
+        if: ${{ matrix.os == 'windows-11-arm' }}
+        run: |
+            curl -LOs https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe
+            ./rustup-init.exe -y --no-modify-path
+            echo "$USERPROFILE/.cargo/bin" >> "$GITHUB_PATH"
       - uses: ./.github/workflows/setup
         with:
           toolchain_flags: "--host ${{ matrix.host_target }}"
 
-      # The `style` job only runs on Linux; this makes sure the Windows-host-specific
-      # code is also covered by clippy.
-      - name: Check clippy
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: ./miri clippy -- -D warnings
+      # We set up the cross-compiler *after* the basic setup as setting CC would otherwise
+      # cause confusion.
+      - name: install gcc-cross
+        if: ${{ matrix.gcc_cross != '' }}
+        run: |
+          sudo apt install gcc-${{ matrix.gcc_cross }}
+          echo "Setting environment variables:"
+          echo "CC_${{ matrix.host_target }}=${{ matrix.gcc_cross }}-gcc" | tee -a $GITHUB_ENV
+          TARGET_UPPERCASE=$(echo ${{ matrix.host_target }} | tr '[:lower:]-' '[:upper:]_')
+          echo "CARGO_TARGET_${TARGET_UPPERCASE}_LINKER=${{ matrix.gcc_cross }}-gcc" | tee -a $GITHUB_ENV
 
-      - name: Test Miri
+      # The main test job! We don't run this in qemu as that is quite slow,
+      # so those targets only get the clippy check below.
+      - name: test Miri
+        if: ${{ !matrix.qemu }}
         run: ./ci/ci.sh
 
+      # The `style` job only runs on Linux; this makes sure the host-specific
+      # code is also covered by clippy.
+      - name: clippy
+        run: ./miri clippy -- -D warnings
+
   style:
     name: style checks
     runs-on: ubuntu-latest
@@ -51,8 +110,6 @@ jobs:
 
       - name: rustfmt
         run: ./miri fmt --check
-      - name: clippy
-        run: ./miri clippy -- -D warnings
       - name: clippy (no features)
         run: ./miri clippy --no-default-features -- -D warnings
       - name: clippy (all features)
@@ -73,7 +130,7 @@ jobs:
   # ALL THE PREVIOUS JOBS NEED TO BE ADDED TO THE `needs` SECTION OF THIS JOB!
   # And they should be added below in `cron-fail-notify` as well.
   conclusion:
-    needs: [build, style, coverage]
+    needs: [test, style, coverage]
     # We need to ensure this job does *not* get skipped if its dependencies fail,
     # because a skipped job is considered a success by GitHub. So we have to
     # overwrite `if:`. We use `!cancelled()` to ensure the job does still not get run
@@ -102,40 +159,53 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 256 # get a bit more of the history
-      - name: install josh-proxy
-        run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
+      - name: install josh-sync
+        run: cargo +stable install --locked --git https://github.com/rust-lang/josh-sync
       - name: setup bot git name and email
         run: |
           git config --global user.name 'The Miri Cronjob Bot'
           git config --global user.email 'miri@cron.bot'
       - name: Install nightly toolchain
         run: rustup toolchain install nightly --profile minimal
-      - name: get changes from rustc
-        run: ./miri rustc-pull
       - name: Install rustup-toolchain-install-master
         run: cargo install -f rustup-toolchain-install-master
-      - name: format changes (if any)
+      - name: Push changes to a branch and create PR
         run: |
+          # Make it easier to see what happens.
+          set -x
+          # Temporarily disable early exit to examine the status code of rustc-josh-sync
+          set +e
+          rustc-josh-sync pull
+          exitcode=$?
+          set -e
+
+          # If there were no changes to pull, rustc-josh-sync returns status code 2.
+          # In that case, skip the rest of the job.
+          if [ $exitcode -eq 2 ]; then
+            echo "Nothing changed in rustc, skipping PR"
+            exit 0
+          elif [ $exitcode -ne 0 ]; then
+            # If return code was not 0 or 2, rustc-josh-sync actually failed
+            echo "error: rustc-josh-sync failed"
+            exit ${exitcode}
+          fi
+
+          # Format changes
           ./miri toolchain
           ./miri fmt --check || (./miri fmt && git commit -am "fmt")
-      - name: Push changes to a branch and create PR
-        run: |
-          # `git diff --exit-code` "succeeds" if the diff is empty.
-          if git diff --exit-code HEAD^; then echo "Nothing changed in rustc, skipping PR"; exit 0; fi
-          # The diff is non-empty, create a PR.
+
+          # Create a PR
           BRANCH="rustup-$(date -u +%Y-%m-%d)"
           git switch -c $BRANCH
           git push -u origin $BRANCH
           gh pr create -B master --title 'Automatic Rustup' --body 'Please close and re-open this PR to trigger CI, then enable auto-merge.'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }}
-          ZULIP_API_TOKEN: ${{ secrets.ZULIP_API_TOKEN }}
 
   cron-fail-notify:
     name: cronjob failure notification
     runs-on: ubuntu-latest
-    needs: [build, style, coverage]
+    needs: [test, style, coverage]
     if: ${{ github.event_name == 'schedule' && failure() }}
     steps:
       # Send a Zulip notification
@@ -153,7 +223,7 @@ jobs:
           It would appear that the [Miri cron job build]('"https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"') failed.
 
           This likely means that rustc changed the miri directory and
-          we now need to do a [`./miri rustc-pull`](https://github.com/rust-lang/miri/blob/master/CONTRIBUTING.md#importing-changes-from-the-rustc-repo).
+          we now need to do a [`rustc-josh-sync pull`](https://github.com/rust-lang/miri/blob/master/CONTRIBUTING.md#importing-changes-from-the-rustc-repo).
 
           Would you mind investigating this issue?
 

.github/workflows/setup/action.yml

@@ -2,6 +2,7 @@ name: "Miri CI setup"
 description: "Sets up Miri CI"
 inputs:
   toolchain_flags:
+    description: extra flags to pass to rustup-toolchain-install-master
     required: false
     default: ''
 runs:
@@ -31,18 +32,15 @@ runs:
             ~/.cargo/bin
             ~/.cargo/.crates.toml
             ~/.cargo/.crates2.json
-          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/**/*.yml') }}
-          restore-keys: cargo-${{ runner.os }}
+          # Bump the version when something here changes that needs a cache reset.
+          key: cargo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}-v1
+          restore-keys: cargo-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Install rustup-toolchain-install-master
+      - name: Install the tools we need
         if: steps.cache.outputs.cache-hit != 'true'
         run: cargo install -f rustup-toolchain-install-master hyperfine
         shell: bash
 
-      - name: Install nightly toolchain
-        run: rustup toolchain install nightly --profile minimal
-        shell: bash
-
       - name: Install "master" toolchain
         run: |
           if [[ ${{ github.event_name }} == 'schedule' ]]; then

.gitignore

@@ -1,5 +1,4 @@
 target
-/doc
 tex/*/out
 *.dot
 *.out
@@ -9,6 +8,9 @@ tex/*/out
 *.mm_profdata
 perf.data
 perf.data.old
-flamegraph.svg
+flamegraph*.svg
+rustc-ice*.txt
 tests/native-lib/libtestlib.so
 .auto-*
+
+/genmc/

CONTRIBUTING.md

@@ -13,18 +13,20 @@ for a list of Miri maintainers.
 
 [Rust Zulip]: https://rust-lang.zulipchat.com
 
-### Pull review process
+### PR review process
 
 When you get a review, please take care of the requested changes in new commits. Do not amend
 existing commits. Generally avoid force-pushing. The only time you should force push is when there
 is a conflict with the master branch (in that case you should rebase across master, not merge), and
 all the way at the end of the review process when the reviewer tells you that the PR is done and you
-should squash the commits. For the latter case, use `git rebase --keep-base ...` to squash without
-changing the base commit your PR branches off of. Use your own judgment and the reviewer's guidance
-to decide whether the PR should be squashed into a single commit or multiple logically separate
-commits. (All this is to work around the fact that Github is quite bad at dealing with force pushes
-and does not support `git range-diff`. Maybe one day Github will be good at git and then life can
-become easier.)
+should squash the commits. (All this is to work around the fact that Github is quite bad at
+dealing with force pushes and does not support `git range-diff`.)
+
+The recommended way to squash commits is to use `./miri squash`, which will make everything into a
+single commit. You will be asked for the commit message; please ensure it describes the entire PR.
+You can also use `git rebase` manually if you need more control (e.g. if there should be more than
+one commit at the end), but then please use `--keep-base` to ensure the PR remains based on the same
+upstream commit.
 
 Most PRs bounce back and forth between the reviewer and the author several times, so it is good to
 keep track of who is expected to take the next step. We are using the `S-waiting-for-review` and
@@ -160,6 +162,15 @@ compiler that has `debug=true` set in `bootstrap.toml`.
 You can set `MIRI_BACKTRACE=1` to get a backtrace of where an
 evaluation error was originally raised.
 
+#### Tracing
+
+You can generate a Chrome trace file from a Miri execution by passing `--features=tracing` during the
+build and then setting `MIRI_TRACING=1` when running Miri. This will generate a `.json` file that
+you can visualize in [Perfetto](https://ui.perfetto.dev/). For example:
+
+```sh
+MIRI_TRACING=1 ./miri run --features=tracing tests/pass/hello.rs
+```
 
 ### UI testing
 
@@ -286,25 +297,27 @@ You can also directly run Miri on a Rust source file:
 
 ## Advanced topic: Syncing with the rustc repo
 
-We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the
+We use the [`josh-sync`](https://github.com/rust-lang/josh-sync) tool to transmit changes between the
 rustc and Miri repositories. You can install it as follows:
 
 ```sh
-cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
+cargo install --locked --git https://github.com/rust-lang/josh-sync
 ```
 
-Josh will automatically be started and stopped by `./miri`.
+The commands below will automatically install and manage the [Josh](https://github.com/josh-project/josh) proxy that performs the actual work.
 
 ### Importing changes from the rustc repo
 
 *Note: this usually happens automatically, so these steps rarely have to be done by hand.*
 
 We assume we start on an up-to-date master branch in the Miri repo.
 
+1) First, create a branch for the pull, e.g. `git checkout -b rustup`
+2) Then run the following:
 ```sh
 # Fetch and merge rustc side of the history. Takes ca 5 min the first time.
 # This will also update the `rustc-version` file.
-./miri rustc-pull
+rustc-josh-sync pull
 # Update local toolchain and apply formatting.
 ./miri toolchain && ./miri fmt
 git commit -am "rustup"
@@ -317,12 +330,12 @@ needed.
 
 ### Exporting changes to the rustc repo
 
-We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
+We will use the `josh-sync` tool to push to your fork of rustc. Run the following in the Miri repo,
 assuming we are on an up-to-date master branch:
 
 ```sh
 # Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
-./miri rustc-push YOUR_NAME miri
+rustc-josh-sync push miri YOUR_NAME
 ```
 
 This will create a new branch called `miri` in your fork, and the output should include a link that
@@ -341,6 +354,7 @@ https. Add the following to your `.gitconfig`:
 
 The following environment variables are relevant to `./miri`:
 
+* `CARGO` sets the binary used to execute Cargo; if none is specified, defaults to `cargo`.
 * `MIRI_AUTO_OPS` indicates whether the automatic execution of rustfmt, clippy and toolchain setup
   (as controlled by the `./auto-*` files) should be skipped. If it is set to `no`, they are skipped.
   This is used to allow automated IDE actions to avoid the auto ops.