diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..183c5fc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +# Exclude everything by default, then include just what we need +# Especially note this means that .git is not included, and not tests/ +# to avoid spurious rebuilds. +* + +# Toplevel build bits +!Makefile +!Cargo.* +# Docs +!docs +# We use the spec file +!packaging/ +# Workaround for podman bug with secrets + remote +# https://github.com/containers/podman/issues/25314 +!podman-build-secret* +# Nextest configuration +!.config/ +# And finally of course all the Rust sources +!crates/ diff --git a/.github/actions/bootc-ubuntu-setup/action.yml b/.github/actions/bootc-ubuntu-setup/action.yml new file mode 100644 index 0000000..eb3f209 --- /dev/null +++ b/.github/actions/bootc-ubuntu-setup/action.yml @@ -0,0 +1,101 @@ +name: 'Bootc Ubuntu Setup' +description: 'Default host setup' +inputs: + libvirt: + description: 'Install libvirt and virtualization stack' + required: false + default: 'false' +runs: + using: 'composite' + steps: + # The default runners have TONS of crud on them... + - name: Free up disk space on runner + shell: bash + run: | + set -xeuo pipefail + sudo df -h + unwanted_pkgs=('^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' + azure-cli google-chrome-stable firefox mono-devel) + unwanted_dirs=(/usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL) + # Start background removal operations as systemd units; if this causes + # races in the future around disk space we can look at waiting for cleanup + # before starting further jobs, but right now we spent a lot of time waiting + # on the network and scripts and such below, giving these plenty of time to run. + n=0 + runcleanup() { + sudo systemd-run -r -u action-cleanup-${n} -- "$@" + n=$(($n + 1)) + } + runcleanup docker image prune --all --force + for x in ${unwanted_dirs[@]}; do + runcleanup rm -rf "$x" + done + # Apt removals in foreground, as we can't parallelize these + for x in ${unwanted_pkgs[@]}; do + /bin/time -f '%E %C' sudo apt-get remove -y $x + done + # We really want support for heredocs + - name: Update podman and install just + shell: bash + run: | + set -eux + # Require the runner is ubuntu-24.04 + IDV=$(. /usr/lib/os-release && echo ${ID}-${VERSION_ID}) + test "${IDV}" = "ubuntu-24.04" + # plucky is the next release + echo 'deb http://azure.archive.ubuntu.com/ubuntu plucky universe main' | sudo tee /etc/apt/sources.list.d/plucky.list + /bin/time -f '%E %C' sudo apt update + # skopeo is currently older in plucky for some reason hence --allow-downgrades + /bin/time -f '%E %C' sudo apt install -y --allow-downgrades crun/plucky podman/plucky skopeo/plucky just + # This is the default on e.g. Fedora derivatives, but not Debian + - name: Enable unprivileged /dev/kvm access + shell: bash + run: | + set -xeuo pipefail + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + # Used by a few workflows, but generally useful + - name: Set architecture variable + id: set_arch + shell: bash + run: echo "ARCH=$(arch)" >> $GITHUB_ENV + # We often use Rust, so set up opinionated default caching + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: true + # Only generate caches on push to git main + save-if: ${{ github.ref == 'refs/heads/main' }} + # Suppress actually using the cache for builds running from + # git main so that we avoid incremental compilation bugs + lookup-only: ${{ github.ref == 'refs/heads/main' }} + # Install libvirt stack if requested + - name: Install libvirt and virtualization stack + if: ${{ inputs.libvirt == 'true' }} + shell: bash + run: | + set -xeuo pipefail + export BCVK_VERSION=0.5.3 + /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system + # Something in the stack is overriding this, but we want session right now for bcvk + echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV + td=$(mktemp -d) + cd $td + # Install bcvk + target=bcvk-$(arch)-unknown-linux-gnu + /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz + tar xzf ${target}.tar.gz + sudo install -T ${target} /usr/bin/bcvk + cd - + rm -rf "$td" + + # Also bump the default fd limit as a workaround for https://github.com/bootc-dev/bcvk/issues/65 + sudo sed -i -e 's,^\* hard nofile 65536,* hard nofile 524288,' /etc/security/limits.conf + - name: Cleanup status + shell: bash + run: | + set -xeuo pipefail + systemctl list-units 'action-cleanup*' + df -h diff --git a/.github/workflows/main-c10s.yml b/.github/workflows/main-c10s.yml new file mode 100644 index 0000000..e171916 --- /dev/null +++ b/.github/workflows/main-c10s.yml @@ -0,0 +1,27 @@ +name: CI (c10s) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + test-ephemeral: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Ubuntu environment with KVM + uses: ./.github/actions/bootc-ubuntu-setup + + - name: Build C10S test container + run: podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s . + + - name: Run ephemeral integration tests + run: | + podman run --rm --privileged --device=/dev/kvm \ + -v bcvk-test-storage:/var/lib/containers \ + localhost/bcvk:c10s diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47b727e..0d76b61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,34 +18,23 @@ jobs: runs-on: ubuntu-24.04 steps: - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y just pkg-config go-md2man libvirt-daemon libvirt-clients qemu-kvm qemu-system qemu-utils virtiofsd + - uses: actions/checkout@v4 - - name: Install podman for heredoc support - run: | - set -eux - echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list - sudo apt update - sudo apt install -y crun/testing podman/testing just + - name: Setup Ubuntu environment + uses: ./.github/actions/bootc-ubuntu-setup + with: + libvirt: 'true' - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - ls -l /dev/kvm + - name: Remove pre-installed bcvk + run: sudo rm -f /usr/bin/bcvk - - uses: actions/checkout@v4 + - name: Install go-md2man for man page generation + run: sudo apt install -y go-md2man - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - - uses: taiki-e/install-action@nextest - - name: Cache build artifacts - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest - name: Build run: just validate && just build diff --git a/Justfile b/Justfile index 739d863..d35ec0f 100644 --- a/Justfile +++ b/Justfile @@ -96,3 +96,10 @@ archive: build install: build cp target/release/bck ~/.local/bin/ +build-image-c10s: + podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s . + +test-ephemeral-c10s: build-image-c10s + # TODO try downgrading to --cap-add=all --security-opt=label=type:container_runtime_t, I think + # we'll need to assume `--net=host` mainly in bcvk in this situation. + podman run --rm --privileged --device=/dev/kvm -v bcvk-test-storage:/var/lib/containers localhost/bcvk:c10s diff --git a/tests/fixtures/Containerfile b/tests/fixtures/Containerfile new file mode 100644 index 0000000..4a49b05 --- /dev/null +++ b/tests/fixtures/Containerfile @@ -0,0 +1,73 @@ +# Container for running bcvk ephemeral tests on C10S +# This mirrors the CI workflow in .github/workflows/main-c10s.yml + +ARG base=quay.io/centos/centos:stream10 +FROM $base as build + +# Install build dependencies +RUN dnf -y install dnf-utils && \ + dnf config-manager --set-enabled crb && \ + dnf install -y pkgconfig go-md2man gcc make openssl-devel openssh-clients && \ + dnf clean all + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install nextest +RUN cargo install cargo-nextest --locked + +# Copy source code +COPY . /src +WORKDIR /src + +# Build bcvk and create integration test archive +RUN make && \ + cargo nextest archive --release -P integration -p integration-tests --archive-file integration-tests.tar.zst + +# Runtime stage +FROM $base + +# Install runtime dependencies for running VMs +RUN dnf -y install dnf-utils && \ + dnf config-manager --set-enabled crb && \ + dnf install -y \ + libvirt-daemon \ + libvirt-daemon-driver-qemu \ + libvirt-client \ + qemu-kvm \ + virtiofsd \ + podman && \ + dnf clean all + +# Copy cargo-nextest from build stage +COPY --from=build /root/.cargo/bin/cargo-nextest /usr/local/bin/cargo-nextest + +# Copy built artifacts +COPY --from=build /src/target/release/bcvk /usr/local/bin/bcvk +COPY --from=build /src/integration-tests.tar.zst /tests/integration-tests.tar.zst + +# Copy source tree metadata needed by nextest +# Nextest needs the workspace structure even when using archives +COPY --from=build /src/Cargo.toml /src/Cargo.lock /tests/ +COPY --from=build /src/.config /tests/.config +COPY --from=build /src/crates /tests/crates + +# Set up environment +ENV BCVK_PATH=/usr/local/bin/bcvk +ENV LIBVIRT_DEFAULT_URI=qemu:///system +WORKDIR /tests + +# Create entrypoint script that pulls images and runs tests +RUN < /usr/local/bin/run-tests.sh +#!/bin/bash +set -euo pipefail +echo "Pulling test images..." +podman pull -q quay.io/fedora/fedora-bootc:42 quay.io/centos-bootc/centos-bootc:stream9 quay.io/centos-bootc/centos-bootc:stream10 +echo "Running ephemeral integration tests..." +exec /usr/local/bin/cargo-nextest nextest run --archive-file integration-tests.tar.zst --workspace-remap /tests ephemeral +EOF +RUN chmod +x /usr/local/bin/run-tests.sh + +# Default command runs the test script +CMD ["/usr/local/bin/run-tests.sh"]