DOCA workflow support (#1469)

Add DOCA workflow support

Co-authored-by: Kayobe Automation <kayobe-automation@example.org>
Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com>
Co-authored-by: Matt Anson <matta@stackhpc.com>

Author: 4 people

.github/workflows/package-build-ofed.yml

@@ -1,5 +1,5 @@
 ---
-name: Build OFED packages
+name: Build OFED kernel modules
 on:
   workflow_dispatch:
     inputs:
@@ -19,11 +19,11 @@ on:
 
 env:
   ANSIBLE_FORCE_COLOR: True
-  KAYOBE_ENVIRONMENT: ci-builder
+  KAYOBE_ENVIRONMENT: ci-doca-builder
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-ofed-packages:
-    name: Build OFED packages
+    name: Build OFED kernel modules
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
@@ -48,6 +48,11 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -86,6 +91,7 @@ jobs:
         id: image_tag
         run: |
           echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name
@@ -145,13 +151,13 @@ jobs:
 
       - name: Write Terraform outputs
         run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-outputs.yml
           ${{ steps.tf_outputs.outputs.stdout }}
           EOF
 
       - name: Write Terraform network config
         run: |
-          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-network-allocation.yml
           ---
           aio_ips:
             builder: "{{ access_ip_v4.value }}"
@@ -176,37 +182,37 @@ jobs:
       - name: Bootstrap the control host
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe control host bootstrap
 
       - name: Run growroot playbook
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Configure the seed host (Builder VM)
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
+          kayobe seed host configure --skip-tags network,docker,docker-registry
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run a distro-sync
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
+          kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Reset BLS entries on the seed host
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reset-bls-entries.yml \
           -e "reset_bls_host=ofed-builder"
         env:
@@ -215,32 +221,33 @@ jobs:
       - name: Disable noexec in /var/tmp
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Reboot to apply the kernel update
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reboot.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run OFED builder playbook
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run OFED upload playbook
         run: |
           source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+          source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

doc/source/contributor/ofed.rst

@@ -4,19 +4,17 @@ OFED
 
 Warning: Experimental workflow subject to change
 
-This section documents the workflow for building OFED packages for Release train integration.
-
-The workflow builds the OFED kernel modules against the latest available kernel in Release train
-(as configured in SKC) and compiles them into RPM packages to be uploaded to Ark. Addtionally,
-this workflow downloads the userspace OFED packages from the Nvidia repository and uploads these
-to Ark.
+The Nvidia DOCA framework is distributed as part of StackHPC Release Train for OFED driver support,
+this repository is synced into Ark as part of the Release Train workflows, however to ensure
+compatibility with Release Train packages, we are required to build OFED modules with support for
+the latest Release Train kernel.
 
 Workflow
 ========
 
 The workflow uses workflow_dispatch to manually request an OFED build, which will deploy a builder
 VM, apply kayobe config to the builder, upgrade the kernel, reboot, then run two Ansible playbooks
-for building and uploading OFED to Ark.
+for building and uploading OFED modules to Ark.
 
 Pre-requisites
 --------------
@@ -25,31 +23,82 @@ Before building OFED packages, the workflow will ensure that:
 
 * A full distro-sync has taken place, ensuring the kernel is upgraded.
 
-* The bootloader has been configured to use the latest kernel
+* The bootloader has been configured to use the latest kernel (reset-bls-entries.yml)
 
 * noexec is disabled in the temporary logical volume.
 
 build-ofed
 ----------
 
-Currently we only support building Rocky Linux 9 OFED packages.
-
-In order to setup OFED, we're required to build kernel modules for the OFED drivers as
-the kernels we provide in release train are unsupported by OFED. To accomplish this we
-will need to use the doca-kernel-support from the doca-extra repository.
+Currently we only support building Rocky Linux 9 OFED kernel module packages.
 
-We will need to instll dependencies in order to build the OFED kernel modules, and these
-are installed at the beginning of the build playbook. We also install base and appstream
-dependencies of userspace OFED packages here, this is intended to stop these dependencies
-being pulled in later when we download the OFED packages from the doca-host repository.
+The Build OFED module workflow will check that the filesystem is configured (noexec disabled)
+to allow the DOCA build script to run. The workflow will also install any necessary dependencies
+for the module build.
 
-At the end of the playbook following the kernel module build, the OFED userspace packages
-are downloaded from the upstream repository in order to upload these to Ark.
+The build script will output a ``doca-kernel-repo`` RPM which contains all kernel modules built
+as part of the workflow. When this RPM is installed, the repofile is created pointing to the
+modules in ``/usr/share/doca-host-<doca-version>/Modules/<kernel-version>/`` on the host.
 
 push-ofed
 ---------
 
-As we're not syncing OFED from any upstream source, and are instead creating our own
-repository of custom packages, we will be required to setup the Pulp distribution/publication
-and upload the content directly to Ark. This playbook uses the Pulp CLI to upload the RPMs
-to Ark.
+As mentioned above, the DOCA repository is synced into the ``doca`` repository in Ark. This workflow
+will upload the ``doca-kernel-repo`` RPM to a separate repository named ``doca-modules``. The version
+for this repository is set in ``pulp-repo-versions.yml`` and is disabled for local pulp syncs by
+default.
+
+Install process
+===============
+
+Release Train configuration
+---------------------------
+
+DOCA repositories will need to be synced to the local Pulp service, Ensure the DOCA
+hosts added to the ``mlnx`` group before running a package sync, if the group is not
+empty DOCA will be synced into the local Pulp. The local Pulp can be synced with Ark
+by running:
+
+.. code-block:: console
+
+  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml
+  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml
+
+DOCA repositories can be templated to hosts by running Kayobe host configure.
+
+.. code-block:: console
+
+  kayobe overcloud host configure -t dnf
+
+StackHPC DOCA kernel modules will require the latest kernel version available in Ark for
+the current Rocky minor version. You should ensure that packages are up to date by running
+a package update, which can also be limited to hosts in the ``mlnx`` group.
+
+.. code-block:: console
+
+  kayobe overcloud host package update --packages "*" --limit mlnx
+
+To ensure the latest kernel is the default on boot, the bootloader entires will need
+to be reset before rebooting.
+
+.. code-block:: console
+
+  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reset-bls-entires.yml -e reset_bls_host=mlnx
+
+The hosts can now be rebooted to use the latest kernel, a rolling reboot may be applicable
+here to reduce distruptions. See the `package updates documentation <package-updates>`.
+
+.. code-block:: console
+
+  kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit mlnx
+
+install-doca
+------------
+
+A playbook is provided to install DOCA on hosts in the ``mlnx`` group. Ensure this group
+is configured to include the hosts you wish to install DOCA on. To run the install
+playbook:
+
+.. code-block:: console
+
+    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/install-doca.yml

etc/kayobe/ansible/build-ofed-rocky.yml

@@ -1,5 +1,5 @@
 ---
-- name: Build OFED packages
+- name: Build OFED kernel modules
   become: true
   hosts: ofed-builder
   gather_facts: false
@@ -17,57 +17,29 @@
     - name: Install package dependencies
       ansible.builtin.dnf:
         name:
-          - kpartx
-          - perl
-          - rpm-build
+          - autoconf
           - automake
-          - patch
-          - kernel
+          - createrepo
+          - cmake-filesystem
+          - doca-extra
+          - gcc-gfortran
           - kernel-devel
-          - autoconf
-          - pciutils
-          - kernel-modules-extra
           - kernel-rpm-macros
-          - lsof
+          - kernel-modules-extra
+          - kpartx
           - libtool
-          - tk
-          - gcc-gfortran
-          - tcl
-          - createrepo
-          - cmake-filesystem
           - libnl3-devel
+          - lsof
+          - patch
+          - pciutils
+          - perl
           - python3-devel
+          - rpm-build
+          - tcl
+          - tk
         state: latest
         update_cache: true
 
-    - name: Add DOCA host repository package
-      ansible.builtin.dnf:
-        name: "https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm"
-        disable_gpg_check: true
-
-    - name: Install DOCA extra packages
-      ansible.builtin.dnf:
-        name: doca-extra
-
-    - name: Create build directory
-      ansible.builtin.file:
-        path: /home/cloud-user/ofed
-        state: directory
-        mode: "0777"
-
-    - name: Set build directory
-      ansible.builtin.replace:
-        path: /opt/mellanox/doca/tools/doca-kernel-support
-        regexp: TMP_DIR=\$1
-        replace: TMP_DIR=/home/cloud-user/ofed
-
     - name: Build OFED kernel modules
-      ansible.builtin.shell:
-        cmd: |
-          /opt/mellanox/doca/tools/doca-kernel-support
-
-    - name: Download OFED userspace packages
-      ansible.builtin.dnf:
-        name: doca-ofed-userspace
-        download_only: true
-        download_dir: /home/cloud-user/ofed
+      ansible.builtin.command:
+        cmd: /opt/mellanox/doca/tools/doca-kernel-support

etc/kayobe/ansible/install-doca.yml

@@ -0,0 +1,28 @@
+---
+- name: Install DOCA
+  become: true
+  hosts: mlnx
+  gather_facts: true
+  tasks:
+    - name: Get running kernel
+      ansible.builtin.command:
+        cmd: "uname -r"
+      register: kernel
+
+    - name: Install kernel repo
+      ansible.builtin.dnf:
+        name: doca-kernel-repo
+        state: latest
+        update_cache: true
+
+    - name: Ensure correct priority for DOCA modules
+      ansible.builtin.lineinfile:
+        line: "priority=-2"
+        insertafter: EOF
+        path: "/etc/yum.repos.d/doca-kernel-{{ kernel.stdout }}.repo"
+
+    - name: Install DOCA OFED
+      ansible.builtin.dnf:
+        name: doca-ofed
+        state: latest
+        update_cache: true