Merge branch 'main' into feat/auto-bump-timestamps

Author: wtripp180901

.github/workflows/extra.yml

@@ -8,12 +8,14 @@ on:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
       - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
       - '.github/workflows/extra.yml'
   pull_request:
     paths:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
       - 'ansible/roles/cuda/**'
+      - 'ansible/roles/lustre/**'
       - '.github/workflows/extra.yml'
 
 jobs:
@@ -29,11 +31,11 @@ jobs:
         build:
           - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
-            inventory_groups: doca,cuda
+            inventory_groups: doca,cuda,lustre
             volume_size: 30 # needed for cuda
           - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
-            inventory_groups: doca,cuda
+            inventory_groups: doca,cuda,lustre
             volume_size: 30 # needed for cuda
     env:
       ANSIBLE_FORCE_COLOR: True

ansible/fatimage.yml

@@ -230,14 +230,6 @@
         name: cloudalchemy.grafana
         tasks_from: install.yml
 
-- hosts: doca
-  become: yes
-  gather_facts: yes
-  tasks:
-    - name: Install NVIDIA DOCA
-      import_role:
-        name: doca
-
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"

ansible/filter_plugins/utils.py

@@ -41,6 +41,9 @@ def to_ood_regex(items):
         eg {{ [compute-001, compute-002, control] | to_regex }} -> '(compute-\d+)|(control)'
     """
 
+    # NB: for python3.12+ the \d in this function & docstring
+    # need to be raw strings. See https://docs.python.org/3/reference/lexical_analysis.html
+
     # There's a python bug which means re.sub() can't use '\d' in the replacement so
     # have to do replacement in two stages:
     r = [re.sub(r"\d+", 'XBACKSLASHX', v) for v in items]

ansible/roles/lustre/README.md

@@ -8,7 +8,7 @@ Install and configure a Lustre client. This builds RPM packages from source.
 
 ## Role Variables
 
-- `lustre_version`: Optional str. Version of lustre to build, default `2.15.5` which is the first version with EL9 support
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6` which is the first version with EL9.5 support
 - `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0`. Only the `tcp` protocol type is currently supported. Default `tcp`.
 - `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`).
 - `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys:

ansible/roles/lustre/defaults/main.yml

@@ -1,4 +1,4 @@
-lustre_version: '2.15.5' # https://www.lustre.org/lustre-2-15-5-released/
+lustre_version: '2.15.6' # https://www.lustre.org/lustre-2-15-6-released/
 lustre_lnet_label: tcp
 #lustre_mgs_nid:
 lustre_mounts: []

ansible/roles/lustre/tasks/install.yml

@@ -41,30 +41,9 @@
   ansible.builtin.dnf:
     name: "{{ _lustre_find_rpms.files | map(attribute='path')}}"
     disable_gpg_check: yes
-  
-- block:
-  - name: Remove lustre build prerequisites
-    # NB Only remove ones this role installed which weren't upgrades
-    ansible.builtin.dnf:
-      name: "{{ _new_pkgs }}"
-      state: absent
-    vars:
-      _installed_pkgs: |
-        {{
-          _lustre_dnf_build_packages.results |
-          select('match', 'Installed:') |
-          map('regex_replace', '^Installed: (.+?)-[0-9].*$', '\1')
-        }}
-      _removed_pkgs: |
-        {{
-          _lustre_dnf_build_packages.results |
-          select('match', 'Removed:') |
-          map('regex_replace', '^Removed: (.+?)-[0-9].*$', '\1')
-        }}
-      _new_pkgs: "{{ _installed_pkgs | difference(_removed_pkgs) }}"
-  
-  - name: Delete lustre build dir
-    file:
-      path: "{{ lustre_build_dir }}"
-      state: absent
+
+- name: Delete lustre build dir
+  file:
+    path: "{{ lustre_build_dir }}"
+    state: absent
   when: lustre_build_cleanup | bool

dev/setup-env.sh

@@ -17,6 +17,7 @@ PYTHON_VERSION=""
 if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
     PYTHON_VERSION="/usr/bin/python3.10"
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
+    # python3.9+ doesn't have selinux bindings
     PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
     PYTHON_VERSION="/usr/bin/python3.9"

requirements.txt

@@ -1,4 +1,4 @@
-ansible==8.0.0
+ansible==6.7.0 # cloudalchemy.prometheus uses ansible.builtin.include, removed in ansible-core==2.16 => ansible==9
 openstacksdk
 python-openstackclient==6.6.1 # v7.0.0 has a bug re. rebuild
 python-manilaclient
@@ -9,5 +9,5 @@ cookiecutter
 selinux # this is a shim to avoid having to use --system-site-packages, you still need sudo yum install libselinux-python3
 netaddr
 matplotlib
-pulp-cli==0.29.2
+pulp-cli==0.23.2
 beautifulsoup4==4.12.3

requirements.yml

@@ -49,6 +49,10 @@ collections:
   - name: https://github.com/azimuth-cloud/ansible-collection-image-utils
     type: git
     version: 0.4.0
+  # stackhpc.pulp has pulp.squeezer as dependency, any version, but latest
+  # requires newer ansible than can install
+  - name: pulp.squeezer
+    version: 0.0.15
   - name: stackhpc.pulp
     version: 0.5.5
 ...