Create tooling for end-to-end testing (#25)

Create two different tools:

  - `test-drive`: A rustc_driver that compiles a crate and run a few sanity checks on StableMIR.
  - `compiletest`: A wrapper to run compiler tests using the `test-drive` tool.

I am also adding a script to run a few rustc tests and a nightly workflow. The files diff is not quite working yet so most tests that fail compilation don't succeed yet.

For our local tests, so far I created two suites (one with a test each): sanity-checks and fixme suite. The fixme suite is for issues that we find that needs fixing.

Author: celinval

.github/scripts/run_own_tests.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Execute our own set of tests using a local `compiletest` tool based on `ui_test`.
+set -e
+set -u
+
+# Where we will store the SMIR tools (Optional).
+TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}"
+# Assume we are inside SMIR repository
+SMIR_PATH=$(git rev-parse --show-toplevel)
+export RUST_BACKTRACE=1
+
+# Build stable_mir tools
+function build_smir_tools() {
+  pushd "${SMIR_PATH}"
+  cargo +nightly build -Z unstable-options --out-dir "${TOOLS_BIN}"
+  export PATH="${TOOLS_BIN}":"${PATH}"
+}
+
+# Run tests
+function run_tests() {
+  SUITES=(
+    "sanity-checks pass"
+    "fixme fix-me"
+  )
+  for suite_cfg in "${SUITES[@]}"; do
+    # Hack to work on older bash like the ones on MacOS.
+    suite_pair=($suite_cfg)
+    suite=${suite_pair[0]}
+    mode=${suite_pair[1]}
+    echo "#### Running suite: ${suite} mode: ${mode}"
+    compiletest \
+        --driver-path="${TOOLS_BIN}/test-drive" \
+        --mode=${mode} \
+        --src-base="tests/${suite}" \
+        --output-dir="target/tests/" \
+        --no-capture
+  done
+}
+
+build_smir_tools
+run_tests

.github/scripts/run_rustc_tests.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Run rustc test suites using our test driver using nightly.
+# This script leverages the rustc's repo compiletest crate.
+#
+# The suites configuration should match:
+# https://github.com/rust-lang/rust/blob/master/src/bootstrap/test.rs
+
+set -e
+set -u
+export RUST_BACKTRACE=1
+
+# Location of a rust repository. Clone one if path doesn't exist.
+RUST_REPO="${RUST_REPO:-"/tmp/rustc"}"
+
+# Where we will store the SMIR tools (Optional).
+TOOLS_BIN="${TOOLS_BIN:-"/tmp/smir/bin"}"
+
+# Assume we are inside SMIR repository
+SMIR_PATH=$(git rev-parse --show-toplevel)
+
+# Build stable_mir tools
+function build_smir_tools() {
+  pushd "${SMIR_PATH}"
+  cargo +nightly build -Z unstable-options --out-dir "${TOOLS_BIN}"
+  export PATH="${TOOLS_BIN}":"${PATH}"
+}
+
+# Set up rustc repository
+function setup_rustc_repo() {
+  if [[ ! -e "${RUST_REPO}" ]]; then
+    mkdir -p "$(dirname ${RUST_REPO})"
+    git clone -b master https://github.com/rust-lang/rust.git "${RUST_REPO}"
+    pushd "${RUST_REPO}"
+    commit="$(rustc +nightly -vV | awk '/^commit-hash/ { print $2 }')"
+    git checkout ${commit}
+    git submodule init -- "${RUST_REPO}/library/stdarch"
+    git submodule update
+  else
+    pushd "${RUST_REPO}"
+  fi
+}
+
+function run_tests() {
+  # Run the following suite configuration for now (test suite + mode)
+  SUITES=(
+    "codegen codegen"
+    "codegen-units codegen-units"
+    # -- The suites below are failing because of fully qualified paths for standard library
+    # E.g.:
+    # -    _10 = _eprint(move _11) -> [return: bb6, unwind unreachable];
+    # +    _10 = std::io::_eprint(move _11) -> [return: bb6, unwind unreachable];
+    #
+    #"ui ui"
+    #"mir-opt mir-opt"
+    #"pretty pretty" -- 2 failing tests
+  )
+
+  SYSROOT=$(rustc +nightly --print sysroot)
+  PY_PATH=$(type -P python3)
+  HOST=$(rustc +nightly -vV | awk '/^host/ { print $2 }')
+  FILE_CHECK="$(which FileCheck-12 || which FileCheck-13 || which FileCheck-14)"
+
+  for suite_cfg in "${SUITES[@]}"; do
+    # Hack to work on older bash like the ones on MacOS.
+    suite_pair=($suite_cfg)
+    suite=${suite_pair[0]}
+    mode=${suite_pair[1]}
+
+    echo "#### Running suite: ${suite} mode: ${mode}"
+    cargo +nightly run -p compiletest -- \
+      --compile-lib-path="${SYSROOT}/lib" \
+      --run-lib-path="${SYSROOT}/lib"\
+      --python="${PY_PATH}" \
+      --rustc-path="${TOOLS_BIN}/test-drive" \
+      --mode=${mode} \
+      --suite="${suite}" \
+      --src-base="tests/${suite}" \
+      --build-base="$(pwd)/build/${HOST}/stage1/tests/${suite}" \
+      --sysroot-base="$SYSROOT" \
+      --stage-id=stage1-${HOST} \
+      --cc= \
+      --cxx= \
+      --cflags= \
+      --cxxflags= \
+      --llvm-components= \
+      --android-cross-path= \
+      --target=${HOST} \
+      --llvm-filecheck="${FILE_CHECK}" \
+      --channel=nightly \
+      --target-rustcflags="--smir-check" \
+      --host-rustcflags="--smir-check"
+  done
+}
+
+build_smir_tools
+setup_rustc_repo
+run_tests

.github/workflows/nightly.yml

@@ -0,0 +1,33 @@
+name: Run compiler tests
+
+on:
+  schedule:
+    - cron: "0 6 * * *"  # Run daily at 06:00 UTC
+  workflow_dispatch:     # Allow manual dispatching
+  pull_request:
+
+jobs:
+  compile-test:
+    name: Rust Compiler Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install latest nightly
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+
+      - name: Test local suites
+        run: ./.github/scripts/run_own_tests.sh
+        env:
+          TOOLS_BIN: "/tmp/smir/bin"
+
+      - name: Test rustc suites
+        run: ./.github/scripts/run_rustc_tests.sh
+        env:
+          RUST_REPO: "/tmp/rustc"
+          TOOLS_BIN: "/tmp/smir/bin"
+        # Don't fail CI for now. See: https://github.com/rust-lang/project-stable-mir/issues/39
+        continue-on-error: true

Cargo.toml

@@ -0,0 +1,12 @@
+# Cargo workspace for utility tools used to check stable-mir in CI
+[workspace]
+resolver = "2"
+members = [
+    "tools/compiletest",
+    "tools/test-drive",
+]
+
+exclude = [
+    "build",
+    "target",
+]

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "nightly"
+components = ["llvm-tools-preview", "rustc-dev", "rust-src", "rustfmt"]

tests/fixme/associated-items/methods.rs

@@ -0,0 +1,83 @@
+//! Example derived from <https://doc.rust-lang.org/reference/items/associated-items.html>
+#![feature(box_into_inner)]
+
+use std::pin::Pin;
+use std::rc::Rc;
+use std::sync::Arc;
+
+#[derive(Default, Debug)]
+struct Example {
+    inner: String,
+}
+
+type Alias = Example;
+trait Trait {
+    type Output;
+}
+impl Trait for Example {
+    type Output = Example;
+}
+
+#[allow(unused)]
+impl Example {
+    pub fn by_value(self: Self) {
+        self.by_ref("by_val");
+    }
+
+    pub fn by_ref(self: &Self, source: &str) {
+        println!("{source}: {}", self.inner);
+    }
+
+    pub fn by_ref_mut(self: &mut Self) {
+        self.inner = "by_ref_mut".to_string();
+        self.by_ref("mut");
+    }
+
+    pub fn by_box(self: Box<Self>) {
+        self.by_ref("by_box");
+        Box::into_inner(self).by_value();
+    }
+
+    pub fn by_rc(self: Rc<Self>) {
+        self.by_ref("by_rc");
+    }
+
+    pub fn by_arc(self: Arc<Self>) {
+        self.by_ref("by_arc");
+    }
+
+    pub fn by_pin(self: Pin<&Self>) {
+        self.by_ref("by_pin");
+    }
+
+    pub fn explicit_type(self: Arc<Example>) {
+        self.by_ref("explicit");
+    }
+
+    pub fn with_lifetime<'a>(self: &'a Self) {
+        self.by_ref("lifetime");
+    }
+
+    pub fn nested<'a>(self: &mut &'a Arc<Rc<Box<Alias>>>) {
+        self.by_ref("nested");
+    }
+
+    pub fn via_projection(self: <Example as Trait>::Output) {
+        self.by_ref("via_projection");
+    }
+
+    pub fn from(name: String) -> Self {
+        Example { inner: name }
+    }
+}
+
+fn main() {
+    let example = Example::from("Hello".to_string());
+    example.by_value();
+
+    let boxed = Box::new(Example::default());
+    boxed.by_box();
+
+    Example::default().by_ref_mut();
+    Example::default().with_lifetime();
+}

tests/sanity-checks/simple/simple_lib.rs

@@ -0,0 +1,23 @@
+//! Just a simple library
+
+pub struct Point {
+    pub x: i64,
+    pub y: i64,
+}
+
+impl Point {
+    pub fn distance(&self, other: &Point) -> u64 {
+        let (x_dist, x_over) = (self.x - other.x).overflowing_pow(2);
+        let (y_dist, y_over) = (self.y - other.y).overflowing_pow(2);
+        if y_over | x_over {
+            panic!("overflow");
+        }
+
+        let dist = (x_dist as u64 + y_dist as u64) >> 1;
+        dist
+    }
+
+    pub fn distance_root(&self) -> u64 {
+        self.distance(&Point { x: 0, y: 0 })
+    }
+}

tools/compiletest/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "compiletest"
+description = "Run tests using compiletest-rs"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+ui_test = "0.20.0"
+clap = { version = "4.1.3", features = ["derive"] }
+
+[package.metadata.rust-analyzer]
+# This crate uses #[feature(rustc_private)].
+# See https://github.com/rust-analyzer/rust-analyzer/pull/7891
+rustc_private = true

tools/compiletest/build.rs

@@ -0,0 +1,16 @@
+use std::env;
+use std::path::PathBuf;
+
+pub fn main() {
+    // Add rustup to the rpath in order to properly link with the correct rustc version.
+    let rustup_home = env::var("RUSTUP_HOME").unwrap();
+    let toolchain = env::var("RUSTUP_TOOLCHAIN").unwrap();
+    let rustc_lib: PathBuf = [&rustup_home, "toolchains", &toolchain, "lib"]
+        .iter()
+        .collect();
+    println!(
+        "cargo:rustc-link-arg-bin=compiletest=-Wl,-rpath,{}",
+        rustc_lib.display()
+    );
+    println!("cargo:rustc-env=RUSTC_LIB_PATH={}", rustc_lib.display());
+}