Merge stackhpc/yoga into stackhpc/zed

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -209,7 +209,7 @@ jobs:
         if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload updated images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Updated images list
           path: /tmp/updated_images.txt
@@ -218,7 +218,7 @@ jobs:
             steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload Rocky 9 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Rocky 9 build logs
           path: |
@@ -228,7 +228,7 @@ jobs:
         if: steps.build_rocky_9.outcome == 'failure'
 
       - name: Upload Ubuntu Jammy 22.04 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Ubuntu Jammy 22.04 build logs
           path: |

.github/workflows/stackhpc-all-in-one.yml

@@ -34,19 +34,19 @@ on:
       vm_interface:
         description: Default network interface name
         type: string
-        default: enp3s0
+        default: ens3
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: general.v1.medium
+        default: en1.large
       vm_network:
         description: Network for the all-in-one VM
         type: string
-        default: stackhpc-release
+        default: stackhpc-ci
       vm_subnet:
         description: Subnet for the all-in-one VM
         type: string
-        default: stackhpc-release-subnet
+        default: stackhpc-ci
       OS_CLOUD:
         description: Name of cloud in clouds.yaml
         type: string
@@ -74,7 +74,7 @@ jobs:
   all-in-one:
     name: All in one
     if: inputs.if
-    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    runs-on: arc-skc-aio-runner
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
@@ -85,6 +85,11 @@ jobs:
       # NOTE(upgrade): Reference the PREVIOUS release branch here.
       PREVIOUS_BRANCH: stackhpc/yoga
     steps:
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: build-essential git unzip nodejs python3 python3-pip python3-wheel python-is-python3
+
       # If testing upgrade, checkout previous release, otherwise checkout current branch
       - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
         uses: actions/checkout@v4
@@ -102,6 +107,10 @@ jobs:
           fi
           echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT
 
+      - name: Make sure dockerd is running and test Docker.
+        run: |
+          docker run --rm hello-world
+
       - name: Output image tag
         id: image_tag
         run: |
@@ -238,11 +247,11 @@ jobs:
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
       - name: Pull current Kayobe image
         run: |
-          sudo docker image pull $KAYOBE_IMAGE
+          docker image pull $KAYOBE_IMAGE
 
       - name: Run growroot
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             ${{ steps.kayobe_image.outputs.kayobe_image }} \
@@ -252,7 +261,7 @@ jobs:
 
       - name: Host configure
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             ${{ steps.kayobe_image.outputs.kayobe_image }} \
@@ -322,7 +331,7 @@ jobs:
 
       - name: Service upgrade
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -334,7 +343,7 @@ jobs:
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
@@ -344,7 +353,7 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
       - name: Upload test result artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' }}
           path: tempest-artifacts/*
@@ -361,8 +370,3 @@ jobs:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: always()
-
-      - name: Prune Docker images over 1 week old
-        # May fail if another prune is running
-        run: sudo docker image prune --all --force --filter until=168h || true
-        if: always()

.github/workflows/stackhpc-build-kayobe-image.yml

@@ -56,20 +56,20 @@ jobs:
           submodules: true
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: |
             image=moby/buildkit:master
@@ -82,15 +82,19 @@ jobs:
            https_proxy: ${{ inputs.https_proxy }}
            no_proxy: ${{ inputs.no_proxy }}
 
+      # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
+      # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./.automation/docker/kayobe/Dockerfile
           context: .
           build-args: |
              http_proxy=${{ inputs.http_proxy }}
              https_proxy=${{ inputs.https_proxy }}
              BASE_IMAGE=${{ inputs.base_image || 'rockylinux:9' }}
+             KAYOBE_USER_UID=1001
+             KAYOBE_USER_GID=1001
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/stackhpc-container-image-build.yml

@@ -40,7 +40,7 @@ jobs:
   generate-tag:
     name: Generate container image tag
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
+    runs-on: ubuntu-latest
     permissions: {}
     outputs:
       datetime_tag: ${{ steps.datetime_tag.outputs.datetime_tag }}
@@ -88,7 +88,7 @@ jobs:
   container-image-build:
     name: Build Kolla container images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
+    runs-on: arc-skc-container-image-builder-runner
     timeout-minutes: 720
     permissions: {}
     strategy:
@@ -97,7 +97,13 @@ jobs:
     needs:
       - generate-tag
     steps:
-      - uses: actions/checkout@v4
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           path: src/kayobe-config
 
@@ -108,20 +114,9 @@ jobs:
           ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
           path: src/kayobe
 
-      - name: Setup networking
+      - name: Make sure dockerd is running and test Docker
         run: |
-          if ! ip l show breth1 >/dev/null 2>&1; then
-              sudo ip l add breth1 type bridge
-          fi
-          sudo ip l set breth1 up
-          if ! ip a show breth1 | grep 192.168.33.3/24; then
-              sudo ip a add 192.168.33.3/24 dev breth1
-          fi
-          if ! ip l show dummy1 >/dev/null 2>&1; then
-              sudo ip l add dummy1 type dummy
-          fi
-          sudo ip l set dummy1 up
-          sudo ip l set dummy1 master breth1
+          docker run --rm hello-world
 
       - name: Install Kayobe
         run: |
@@ -132,23 +127,19 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      - name: Bootstrap the control host
+      # Required for Docker registry login. Normally installed during host configure.
+      - name: Install Docker Python SDK
         run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe control host bootstrap
-
-      - name: Configure the seed host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+          pip install --user docker
 
-      - name: Prune local Kolla container images over 1 week old
+      - name: Configure localhost as a seed
         run: |
-          sudo docker image prune --all --force --filter until=168h --filter="label=kolla_version"
+          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
+          # A 'seed' host used for building images.
+          # Use localhost for container image builds.
+          [seed]
+          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
+          EOF
 
       - name: Build and push kolla overcloud images
         run: |
@@ -183,13 +174,13 @@ jobs:
 
       - name: Get built container images
         run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images
+          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:*${{ matrix.distro }}*${{ needs.generate-tag.outputs.datetime_tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Upload container images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.distro }} container images
           path: ${{ matrix.distro }}-container-images

.github/workflows/stackhpc-pull-request.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Check changed files
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           # Filters are defined in this file.
@@ -51,7 +51,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python ${{ matrix.python-version }} 🐍
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Tox 📦
@@ -98,7 +98,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -115,7 +115,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -132,7 +132,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -149,7 +149,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -168,7 +168,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
       upgrade: true
     secrets: inherit
@@ -186,7 +186,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
       upgrade: true
     secrets: inherit

etc/kayobe/apt.yml

@@ -75,6 +75,22 @@ apt_repositories: "{{ stackhpc_apt_repositories if 'overcloud' in group_names el
 # repository mirrors are deployed.
 apt_disable_sources_list: "{{ 'overcloud' in group_names }}"
 
+# Apt auth configuration for accessing the package repository mirror.
+stackhpc_apt_auth:
+  - machine: "{{ stackhpc_repo_mirror_url }}"
+    login: "{{ stackhpc_repo_mirror_username }}"
+    password: "{{ stackhpc_repo_mirror_password }}"
+    filename: "kayobe.conf"
+
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in which to store the auth configuration. The
+#   extension should be '.conf'.
+# Default is an empty list.
+apt_auth: "{{ stackhpc_apt_auth if 'overcloud' in group_names and stackhpc_repo_mirror_username is truthy else [] }}"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes