Migrates aarch64 CI runners to Ubuntu (#385)

BradyBonnette · web-flow · commit d39c3b740f73 · 2023-09-19T10:24:51.000-04:00
Ubuntu generally does not require workarounds or tricks to get Postgres
+ PL/Rust to play nicely, so we are going to move the CI runners to use
Ubuntu 22.04 instead of AL2. This should make future updates a little
less painless.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,95 +13,134 @@ on:
 
 env:
   RUST_BACKTRACE: 1
+  SCCACHE_VER: 0.5.4 # Change this to bump the sccache version across all jobs
   # CARGO_LOG: cargo::core::compiler::fingerprint=info # Uncomment this to output compiler fingerprint info
 
 jobs:
-  # A job for running tasks on an Amazon Linux 2 instance, which are self-hosted runners on another cloud service.
-  # The image created for running these tasks already have a majority of the dependencies installed, such as system
-  # libraries, sccache, github runners, rust, etc. This is to cut back on the amount of time it takes to launch a runner.
-  # If something is missing, it can be added here or baked into a new image outside of Github Actions.
-  plrust_arm64_amzn2:
-    name: aarch64 tests (amzn2)
-    runs-on: [self-hosted, linux, ARM64, launch_template_id__lt-0013395587950cb5d]
+  # This job runs all tasks within an Ubuntu 22.04 image launched in AWS.
+  # A few things to note about the environment in which these run:
+  #   * Ubuntu AMIs in AWS require extra storage to be mounted via EBS. As such, EBS volumes are pre-mounted
+  #     in these images under /workdir
+  #   * /workdir is the root directory for almost all things CI-build related, such as:
+  #     - The "ubuntu" user's home directory is relocated to /workdir/ubuntu
+  #     - TMPDIR is now set to /workdir/tmp so that Rust builds won't run out of drive space
+  #     - rustup/cargo/etc are now located under various subdirectories of /workdir/ubuntu
+  #     - sccache directory is located under /workdir/sccache
+  #     - Any artifacts that need to be produces (like cache archives) need to be put under /workdir/ubuntu/artifacts
+  #     - Github Actions runner is downloaded and installed to /workdir/ubuntu/actions-runner -- this is also where
+  #       the runner application will be configured and launched
+  #   * Caches are stored to and pulled from private S3 buckets
+  plrust_arm64_ubuntu:
+    name: arm64 tests on Ubuntu
+
+    runs-on: [self-hosted, linux, ARM64, launch_template_id__lt-0726d9ff7411af069]
     defaults:
       run:
         shell: bash
 
     strategy:
       matrix:
-        version: ["pg13", "pg14", "pg15"]
+        version: ["pg13", "pg14", "pg15", "pg16"]
         target: ["host", "postgrestd"]
       fail-fast: false
 
-    # Note about environment variables: Even though a majority of these environment variables are baked into the runner image,
-    # Github Actions seemingly runs self-hosted tasks in a different manner such that even baken-in environment variables are
-    # removed. Howevever, they can be declared here and persist through all tasks. Assume that the home directory for the
-    # runner is /home/ec2-user
     env:
+      ARTIFACTS_DIR: /workdir/ubuntu/artifacts
       AWS_CACHE_BUCKET: tcdi-ci-plrust-build-cache.private
       CACHE_KEY_VERSION: v0
       CI: true
-      PLRUST_TRUSTED_PGRX_OVERRIDE: "pgrx = { path = '/home/ec2-user/actions-runner/_work/plrust/plrust/plrust-trusted-pgrx', package='plrust-trusted-pgrx' }"
-      RUSTUP_HOME: /home/ec2-user/.rustup
+      PLRUST_TRUSTED_PGRX_OVERRIDE: "pgrx = { path = '/workdir/ubuntu/actions-runner/_work/plrust/plrust/plrust-trusted-pgrx', package='plrust-trusted-pgrx' }"
+      RUSTUP_HOME: /workdir/ubuntu/.rustup
       RUSTC_WRAPPER: sccache
       RUSTFLAGS: -Copt-level=0 -Dwarnings
-      SCCACHE_BIN_DIR: /home/ec2-user/.local/bin
+      SCCACHE_BIN_DIR: /workdir/ubuntu/.local/bin
       SCCACHE_CACHE_SIZE: 20G
-      SCCACHE_DIR: /home/ec2-user/.cache/sccache
+      SCCACHE_DIR: /workdir/sccache
       SCCACHE_IDLE_TIMEOUT: 0
-      WORK_DIR: /home/ec2-user/actions-runner/_work/plrust/plrust
+      WORK_DIR: /workdir/ubuntu/actions-runner/_work/plrust/plrust
+      TMPDIR: /workdir/tmp
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Generate cache filename checksum
+    - name: Generate cache filenames
       run: |
         cd $WORK_DIR
         shopt -s globstar
         checksum=$(cat **/Cargo.lock **/rust-toolchain.toml .github/workflows/ci.yml .cargo/config | sha256sum | awk '{print $1}')
         echo "CACHE_KEY_CHECKSUM=$checksum" >> $GITHUB_ENV
+        echo "CARGO_CACHE_KEY=plrust-arm64-ubuntu-cargo-cache-$CACHE_KEY_VERSION-$checksum.tar.lz4" >> $GITHUB_ENV
+        echo "SCCACHE_CACHE_KEY=plrust-arm64-ubuntu-sccache-cache-$CACHE_KEY_VERSION-$checksum.tar.lz4" >> $GITHUB_ENV
+        mkdir -p $ARTIFACTS_DIR
 
     - name: Set up (Linux) prerequisites and environment
       run: |
         echo ""
-        echo "----- Print kernel info -----"
-        uname -a
-        echo ""
 
         echo "----- Set up dynamic variables -----"
         export PG_VER=$(echo ${{ matrix.version }} | cut -c 3-)
         echo "PG_VER=$PG_VER" >> $GITHUB_ENV
         echo "MAKEFLAGS=$MAKEFLAGS -j $(grep -c ^processor /proc/cpuinfo)" >> $GITHUB_ENV
-        echo "$SCCACHE_BIN_DIR" >> $GITHUB_PATH
-        echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
+        cat $GITHUB_ENV
         echo ""
 
-        echo "----- Install system dependencies -----"
-        # Add any extra dependencies here if necessary
 
-        echo "----- Install PostgreSQL $PG_VER -----"
-        sudo yum install -y "postgresql$PG_VER" "postgresql$PG_VER-server" "postgresql$PG_VER-devel"
-        export PATH="/usr/pgsql-$PG_VER/bin/:$PATH"
-        echo "/usr/pgsql-$PG_VER/bin/" >> $GITHUB_PATH
+        echo "----- Install sccache -----"
+        mkdir -p $SCCACHE_BIN_DIR
+        curl -L "https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VER/sccache-v$SCCACHE_VER-aarch64-unknown-linux-musl.tar.gz" | tar xz
+        mv -f "sccache-v$SCCACHE_VER-aarch64-unknown-linux-musl/sccache" "$SCCACHE_BIN_DIR/sccache"
+        chmod +x "$SCCACHE_BIN_DIR/sccache"
+        echo "$SCCACHE_BIN_DIR" >> "$GITHUB_PATH"
+        mkdir -p "$SCCACHE_DIR"
+        echo ""
 
-        echo "----- Set postgres permissions -----"
-        sudo chmod a+rwx `$(which pg_config) --pkglibdir` `$(which pg_config) --sharedir`/extension
+        echo "----- Remove old postgres and libraries -----"
+        sudo apt remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' # '^mono-llvm.*'
         echo ""
 
-        echo "----- Create artifacts directory -----"
-        mkdir -p $HOME/artifacts
+        echo "----- Set up PostgreSQL Apt repository -----"
+        sudo apt-get install -y wget gnupg
+        sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+        sudo apt-get update -y -qq --fix-missing
         echo ""
 
-        cat $GITHUB_ENV
+        echo "----- Install system dependencies and PostgreSQL version $PG_VER -----"
+        sudo apt-get install -y \
+          build-essential \
+          clang-14 \
+          gcc \
+          libclang-14-dev \
+          libssl-dev \
+          libz-dev \
+          llvm-14-dev \
+          lz4 \
+          make \
+          pkg-config \
+          strace \
+          zlib1g-dev
         echo ""
+
+        echo "----- Install rustup -----"
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+        source "$HOME/.cargo/env"
+        cargo --version
+        echo ""
+
+        echo "----- Print env -----"
         env
+        echo ""
 
-    - name: Load Cargo cache if available
+    - name: Load Cargo and sccache caches if available
       run: |
         # See <plrust-root>/.github/scripts/load_cache.sh for more details
         . $WORK_DIR/.github/scripts/load_cache.sh
-        cargo_cache_key="plrust-arm64-amzn2-cargo-cache-$CACHE_KEY_VERSION-$CACHE_KEY_CHECKSUM.tar.lz4"
-        loadcache $cargo_cache_key
+        loadcache $CARGO_CACHE_KEY
+        loadcache $SCCACHE_CACHE_KEY
+
+    - name: Start sccache server
+      run: sccache --start-server && sccache --show-stats
 
     - name: Create protected files
       run: |
@@ -111,20 +150,22 @@ jobs:
         sudo chmod -R 600 /var/ci-stuff/secret_rust_files
       if: matrix.target == 'postgrestd'
 
-    - name: Load sccache cache if available
+    - name: Install release version of PostgreSQL
       run: |
-        # See <plrust-root>/.github/scripts/load_cache.sh for more details
-        . $WORK_DIR/.github/scripts/load_cache.sh
-        sccache_key="plrust-arm64-amzn2-sccache-cache-$CACHE_KEY_VERSION-$CACHE_KEY_CHECKSUM.tar.lz4"
-        loadcache $sccache_key
+        sudo apt-get install -y \
+          postgresql-"$PG_VER" \
+          postgresql-server-dev-"$PG_VER"
 
-    - name: Start sccache server
-      run: sccache --start-server && sccache --show-stats
+        echo "---- pg_config info ----"
+        /usr/lib/postgresql/"$PG_VER"/bin/pg_config
+
+    - name: Set up Postgres permissions
+      run: sudo chmod a+rwx "$(/usr/lib/postgresql/"$PG_VER"/bin/pg_config --pkglibdir)" "$(/usr/lib/postgresql/"$PG_VER"/bin/pg_config --sharedir)"/extension /var/run/postgresql/
 
     # See <plrust-root>/.github/scripts/install_cargo_pgrx.sh for more details
     - name: Install cargo-pgrx defined in plrust/Cargo.toml
       run: |
-        . $WORK_DIR/.github/scripts/install_cargo_pgrx.sh
+        . $GITHUB_WORKSPACE/.github/scripts/install_cargo_pgrx.sh
         install_cargo_pgrx
 
     - name: Print sccache stats
@@ -133,19 +174,33 @@ jobs:
     - name: Install llvm-tools-preview
       run: rustup component add llvm-tools-preview rustc-dev
 
+    - name: Create protected files
+      run: |
+        sudo mkdir -p /var/ci-stuff/secret_rust_files
+        sudo echo "const FOO:i32 = 7;" /var/ci-stuff/secret_rust_files/const_foo.rs
+        sudo echo "const BAR:i32 = 8;" /var/ci-stuff/const_bar.rs
+        sudo chmod -R 600 /var/ci-stuff/secret_rust_files
+      if: matrix.target == 'postgrestd'
+
     - name: Test plrustc
       run: cd plrustc && cargo test -p plrustc
 
     - name: Print sccache stats
       run: sccache --show-stats
 
-    - name: install plrustc
+    - name: Test plrustc
+      run: cd plrustc && cargo test
+
+    - name: Print sccache stats
+      run: sccache --show-stats
+
+    - name: Install plrustc
       run: cd plrustc && ./build.sh && cp ../build/bin/plrustc ~/.cargo/bin
 
     - name: Print sccache stats
       run: sccache --show-stats
 
-    - name: Run cargo pgrx init
+    - name: Run 'cargo pgrx init' against system-level ${{ matrix.version }}
       run: cargo pgrx init --pg$PG_VER $(which pg_config)
 
     - name: Install PL/Rust as "trusted"
@@ -172,29 +227,23 @@ jobs:
       if: matrix.target == 'host'
       run: cd plrust && echo "\q" | cargo pgrx run "pg$PG_VER" && cd ../plrust-tests && cargo test --no-default-features --features "pg$PG_VER"
 
-    - name: Print sccache stats (after build)
+    - name: Print sccache stats
       run: sccache --show-stats
 
     - name: Stop sccache server
       run: sccache --stop-server || true
 
-    - name: Store Cargo cache if applicable
+    - name: Store Cargo and sccache caches if applicable
       run: |
         . $WORK_DIR/.github/scripts/save_cache.sh
         # See <plrust-root>/.github/scripts/save_cache.sh for more details
-        cargo_cache_key="plrust-arm64-amzn2-cargo-cache-$CACHE_KEY_VERSION-$CACHE_KEY_CHECKSUM.tar.lz4"
         cargo_dirs=( \
             $HOME/.cargo/ \
         )
-        savecache $cargo_cache_key "${cargo_dirs[@]}"
+        savecache $CARGO_CACHE_KEY "${cargo_dirs[@]}"
 
-    - name: Store sccache if applicable
-      run: |
-        # See <plrust-root>/.github/scripts/save_cache.sh for more details
-        . $WORK_DIR/.github/scripts/save_cache.sh
-        sccache_key="plrust-arm64-amzn2-sccache-cache-$CACHE_KEY_VERSION-$CACHE_KEY_CHECKSUM.tar.lz4"
         sccache_dirs=($SCCACHE_DIR)
-        savecache $sccache_key "${sccache_dirs[@]}"
+        savecache $SCCACHE_CACHE_KEY "${sccache_dirs[@]}"
 
   plrust_x86_64:
     name: x86_64 tests
@@ -234,12 +283,12 @@ jobs:
 
 
         echo "----- Install sccache -----"
-        mkdir -p $SCCACHE_BIN_DIR
-        curl -L https://github.com/mozilla/sccache/releases/download/v0.5.0/sccache-v0.5.0-x86_64-unknown-linux-musl.tar.gz | tar xz
-        mv -f sccache-v0.5.0-x86_64-unknown-linux-musl/sccache $SCCACHE_BIN_DIR/sccache
-        chmod +x $SCCACHE_BIN_DIR/sccache
+        mkdir -p "$SCCACHE_BIN_DIR"
+        curl -L "https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VER/sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl.tar.gz" | tar xz
+        mv -f "sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl/sccache" "$SCCACHE_BIN_DIR/sccache"
+        chmod +x "$SCCACHE_BIN_DIR/sccache"
         echo "$SCCACHE_BIN_DIR" >> $GITHUB_PATH
-        mkdir -p $SCCACHE_DIR
+        mkdir -p "$SCCACHE_DIR"
         echo ""
 
         echo "----- Remove old postgres -----"
@@ -269,7 +318,7 @@ jobs:
         echo "----- Installed Packages -----"
         sudo apt list --installed
         echo ""
-        
+
         echo "----- Print env -----"
         env
         echo ""
@@ -290,7 +339,7 @@ jobs:
         sudo apt-get install -y \
           postgresql-"$PG_VER" \
           postgresql-server-dev-"$PG_VER"
-        
+
         echo "---- pg_config info ----"
         /usr/lib/postgresql/"$PG_VER"/bin/pg_config
 
