Generalise host image playbooks (#1390)

* Generalise image upload playbook

* Replace occurences to host-image

* Variable for file extension

* Revert changes to overcloud image build

* Update file extension variable

* Use image name

* s/image/artifact/g

* Remove image_repository creds

* Fix artifact references in host-image-build

* Generalise promotion workflow

Author: assumptionsandg

.github/workflows/overcloud-host-image-build.yml

@@ -239,9 +239,11 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path=/opt/kayobe/images/overcloud-rocky-9 \
-          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/overcloud-rocky-9 \
+          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          -e artifact_type="kayobe-images" \
+          -e file_regex="*.qcow2" \
           -e os_distribution="rocky" \
           -e os_release="9"
         env:
@@ -291,9 +293,11 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
-          -e image_path=/opt/kayobe/images/overcloud-ubuntu-jammy \
-          -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-jammy \
+          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          -e artifact_type="kayobe-images" \
+          -e file_regex="*.qcow2" \
           -e os_distribution="ubuntu" \
           -e os_release="jammy"
         env:

.github/workflows/overcloud-host-image-promote.yml

@@ -76,12 +76,12 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \
-          -e image_path='/opt/kayobe/images/overcloud-rocky-linux-9' \
+          src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \
+          -e artifact_type="kayobe-images" \
           -e os_distribution='rocky' \
           -e os_release='9'
         env:
-          OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }}
+          ARTIFACT_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9
 
@@ -90,11 +90,11 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \
-          -e image_path='/opt/kayobe/images/overcloud-ubuntu-jammy' \
+          src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \
+          -e artifact_type="kayobe-images" \
           -e os_distribution='ubuntu' \
           -e os_release='jammy'
         env:
-          OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }}
+          ARTIFACT_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu-jammy

etc/kayobe/ansible/pulp-host-image-promote.yml renamed to etc/kayobe/ansible/pulp-artifact-promote.yml

@@ -1,31 +1,30 @@
 ---
-# Tag is the version of the image to promote, stored in an env variable named OVERCLOUD_HOST_IMAGE_TAG
-- name: Promote an image to production
+# Tag is the version of the artifact to promote, stored in an env variable named ARTIFACT_TAG
+- name: Promote an artifact to production
   hosts: localhost
   vars:
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
-    remote_pulp_username: "{{ stackhpc_image_repository_username }}"
-    remote_pulp_password: "{{ stackhpc_image_repository_password }}"
-    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
-    base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
-    promotion_tag: "{{ lookup('env', 'OVERCLOUD_HOST_IMAGE_TAG') }}"
+    remote_pulp_username: "{{ stackhpc_release_pulp_username }}"
+    remote_pulp_password: "{{ stackhpc_release_pulp_password }}"
+    repository_name: "{{ artifact_type }}-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
+    pulp_base_path: "{{ artifact_type }}/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
+    promotion_tag: "{{ lookup('env', 'ARTIFACT_TAG') }}"
   tasks:
-
-    - name: Check whether the image exists
+    - name: Check whether the artifact exists
       pulp.squeezer.file_distribution:
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_{{ promotion_tag }}"
-        base_path: "{{ base_path }}/{{ promotion_tag }}"
+        base_path: "{{ pulp_base_path }}/{{ promotion_tag }}"
       register: distribution_details
       until: distribution_details is success
       retries: 3
       delay: 5
 
-    - name: Fail if the image does not exist
+    - name: Fail if the artifact does not exist
       fail:
-        msg: "Image {{ promotion_tag }} does not exist"
+        msg: "artifact {{ promotion_tag }} does not exist"
       when: distribution_details.distribution is none
 
     - name: Ensure production content guard is set
@@ -34,7 +33,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_{{ promotion_tag }}"
-        base_path: "{{ base_path }}/{{ promotion_tag }}"
+        base_path: "{{ pulp_base_path }}/{{ promotion_tag }}"
         content_guard: release
         state: present
       register: content_guard_result

etc/kayobe/ansible/pulp-host-image-upload.yml renamed to etc/kayobe/ansible/pulp-artifact-upload.yml

@@ -1,21 +1,21 @@
 ---
-- name: Upload and create a distribution for an image
+- name: Upload and create a distribution for an artifact
   hosts: seed
   vars:
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
-    remote_pulp_username: "{{ stackhpc_image_repository_username }}"
-    remote_pulp_password: "{{ stackhpc_image_repository_password }}"
-    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
-    pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
+    remote_pulp_username: "{{ stackhpc_release_pulp_username }}"
+    remote_pulp_password: "{{ stackhpc_release_pulp_password }}"
+    repository_name: "{{ artifact_type }}-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
+    pulp_base_path: "{{ artifact_type }}/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
-    - name: Print image tag
+    - name: Print artifact tag
       debug:
-        msg: "Image tag: {{ host_image_tag }}"
+        msg: "artifact tag: {{ artifact_tag }}"
 
     - name: Get filename
       find:
-        paths: "{{ image_path }}"
-        patterns: '*.qcow2'
+        paths: "{{ artifact_path }}"
+        patterns: "{{ file_regex }}"
       register: found_files
 
     - name: Upload an artifact
@@ -107,8 +107,8 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ repository_name }}_{{ host_image_tag }}"
-        base_path: "{{ pulp_base_path }}/{{ host_image_tag }}"
+        name: "{{ repository_name }}_{{ artifact_tag }}"
+        base_path: "{{ pulp_base_path }}/{{ artifact_tag }}"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -118,24 +118,24 @@
       retries: 3
       delay: 5
 
-    - name: Update new images file with versioned path
+    - name: Update new artifacts file with versioned path
       lineinfile:
-        path: /tmp/updated_images.txt
+        path: /tmp/updated_artifacts.txt
         line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
-        {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
+        {{ artifact_tag }}/{{ found_files.files[0].path | basename }}"
         create: true
 
-    - name: Update new images file with latest path
+    - name: Update new artifacts file with latest path
       lineinfile:
-        path: /tmp/updated_images.txt
+        path: /tmp/updated_artifacts.txt
         line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print versioned path
       debug:
         msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
-        {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
+        {{ artifact_tag }}/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print latest path
@@ -146,5 +146,5 @@
 
     - name: Print version tag
       debug:
-        msg: "New tag: {{ host_image_tag }}"
+        msg: "New tag: {{ artifact_tag }}"
       when: latest_distribution_details.changed

etc/kayobe/ansible/pulp-host-image-download.yml

@@ -25,8 +25,8 @@
     - name: Download image artifact
       get_url:
         url: "{{ stackhpc_overcloud_host_image_url_no_auth }}"
-        username: "{{ stackhpc_image_repository_username }}"
-        password: "{{ stackhpc_image_repository_password }}"
+        username: "{{ stackhpc_release_pulp_username }}"
+        password: "{{ stackhpc_release_pulp_password }}"
         force_basic_auth: true
         unredirected_headers:
           - "Authorization"

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -94,5 +94,5 @@ stackhpc_docker_registry_password: !vault |
   3664
 
 # Username and password of the overcloud host image repository.
-stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
-stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"
+stackhpc_release_pulp_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_docker_registry_password }}"