Merge pull request #917 from stackhpc/CleanAiOChanges

CI: Use ARC runners for aio jobs, move CI to Leafcloud

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -362,7 +362,7 @@ jobs:
         if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload updated images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Updated images list
           path: /tmp/updated_images.txt
@@ -374,7 +374,7 @@ jobs:
             steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload CentOS build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: CentOS build logs
           path: |
@@ -384,7 +384,7 @@ jobs:
         if: steps.build_centos_stream_8.outcome == 'failure'
 
       - name: Upload Rocky 8 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Rocky 8 build logs
           path: |
@@ -394,7 +394,7 @@ jobs:
         if: steps.build_rocky_8.outcome == 'failure'
 
       - name: Upload Rocky 9 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Rocky 9 build logs
           path: |
@@ -404,7 +404,7 @@ jobs:
         if: steps.build_rocky_9.outcome == 'failure'
 
       - name: Upload Ubuntu Focal 20.04 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Ubuntu Focal 20.04 build logs
           path: |
@@ -414,7 +414,7 @@ jobs:
         if: steps.build_ubuntu_focal.outcome == 'failure'
 
       - name: Upload Ubuntu Jammy 22.04 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Ubuntu Jammy 22.04 build logs
           path: |

.github/workflows/stackhpc-all-in-one.yml

@@ -34,19 +34,19 @@ on:
       vm_interface:
         description: Default network interface name
         type: string
-        default: enp3s0
+        default: ens3
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: general.v1.medium
+        default: en1.large
       vm_network:
         description: Network for the all-in-one VM
         type: string
-        default: stackhpc-release
+        default: stackhpc-ci
       vm_subnet:
         description: Subnet for the all-in-one VM
         type: string
-        default: stackhpc-release-subnet
+        default: stackhpc-ci
       OS_CLOUD:
         description: Name of cloud in clouds.yaml
         type: string
@@ -70,17 +70,26 @@ jobs:
   all-in-one:
     name: All in one
     if: inputs.if
-    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    runs-on: arc-skc-aio-runner
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: build-essential git unzip nodejs python3 python3-pip python3-wheel python-is-python3
+
       - uses: actions/checkout@v4
         with:
            submodules: true
 
+      - name: Make sure dockerd is running and test Docker.
+        run: |
+          docker run --rm hello-world
+
       - name: Output image tag
         id: image_tag
         run: |
@@ -209,11 +218,11 @@ jobs:
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
       - name: Pull latest Kayobe image
         run: |
-          sudo docker image pull $KAYOBE_IMAGE
+          docker image pull $KAYOBE_IMAGE
 
       - name: Run growroot
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -223,7 +232,7 @@ jobs:
 
       - name: Host configure
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -233,7 +242,7 @@ jobs:
 
       - name: Service deploy
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -243,7 +252,7 @@ jobs:
 
       - name: Configure aio resources
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -254,7 +263,7 @@ jobs:
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
@@ -264,7 +273,7 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
       - name: Upload test result artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
           path: tempest-artifacts/*
@@ -284,5 +293,5 @@ jobs:
 
       - name: Prune Docker images over 1 week old
         # May fail if another prune is running
-        run: sudo docker image prune --all --force --filter until=168h || true
+        run: docker image prune --all --force --filter until=168h || true
         if: always()

.github/workflows/stackhpc-build-kayobe-image.yml

@@ -52,20 +52,20 @@ jobs:
            submodules: true
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: |
             image=moby/buildkit:master
@@ -78,14 +78,18 @@ jobs:
            https_proxy: ${{ inputs.https_proxy }}
            no_proxy: ${{ inputs.no_proxy }}
 
+      # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
+      # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./.automation/docker/kayobe/Dockerfile
           context: .
           build-args: |
              http_proxy=${{ inputs.http_proxy }}
              https_proxy=${{ inputs.https_proxy }}
+             KAYOBE_USER_UID=1001
+             KAYOBE_USER_GID=1001
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/stackhpc-container-image-build.yml

@@ -209,7 +209,7 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Upload container images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.distro }} container images
           path: ${{ matrix.distro }}-container-images

.github/workflows/stackhpc-pull-request.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Check changed files
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           # Filters are defined in this file.
@@ -51,7 +51,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python ${{ matrix.python-version }} 🐍
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Tox 📦
@@ -85,7 +85,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -99,7 +99,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -115,7 +115,7 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -131,7 +131,7 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -148,7 +148,7 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -165,7 +165,7 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -182,7 +182,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -199,7 +199,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -216,7 +216,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -233,7 +233,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

etc/kayobe/apt.yml

@@ -75,6 +75,22 @@ apt_repositories: "{{ stackhpc_apt_repositories if 'overcloud' in group_names el
 # repository mirrors are deployed.
 apt_disable_sources_list: "{{ 'overcloud' in group_names }}"
 
+# Apt auth configuration for accessing the package repository mirror.
+stackhpc_apt_auth:
+  - machine: "{{ stackhpc_repo_mirror_url }}"
+    login: "{{ stackhpc_repo_mirror_username }}"
+    password: "{{ stackhpc_repo_mirror_password }}"
+    filename: "kayobe.conf"
+
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in which to store the auth configuration. The
+#   extension should be '.conf'.
+# Default is an empty list.
+apt_auth: "{{ stackhpc_apt_auth if 'overcloud' in group_names and stackhpc_repo_mirror_username is truthy else [] }}"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes