Merge branch 'main' into fix/templating2

Author: sjpb

.github/workflows/stackhpc.yml

@@ -16,6 +16,10 @@ jobs:
       fail-fast: false # as want clouds to continue independently
     concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
+    env:
+      ANSIBLE_FORCE_COLOR: True
+      OS_CLOUD: openstack
+      TF_VAR_cluster_name: ci${{ github.run_id }}
     steps:
       - uses: actions/checkout@v2
 
@@ -26,7 +30,6 @@ jobs:
           echo "${${{ matrix.cloud }}_SSH_KEY}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
         env:
-          smslabs_SSH_KEY: ${{ secrets.SSH_KEY }}
           arcus_SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
 
       - name: Add bastion's ssh key to known_hosts
@@ -49,19 +52,14 @@ jobs:
           echo "${${{ matrix.cloud }}_CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml
         shell: bash
         env:
-          smslabs_CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
 
       - name: Provision ports, inventory and other infrastructure apart from nodes
-        id: provision_ports
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           TF_VAR_create_nodes=false terraform apply -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
       
       - name: Setup environment-specific inventory/terraform inputs
         run: |
@@ -71,7 +69,6 @@ jobs:
           echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           ansible-playbook ansible/adhoc/template-cloud-init.yml
         env:
-          ANSIBLE_FORCE_COLOR: True
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Provision servers
@@ -81,9 +78,6 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
 
       - name: Get server provisioning failure messages
         id: provision_failure
@@ -94,9 +88,6 @@ jobs:
           TF_FAIL_MSGS="$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
           echo TF failure messages: $TF_FAIL_MSGS
           echo "::set-output name=messages::${TF_FAIL_MSGS}"
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: always() && steps.provision_servers.outcome == 'failure'
 
       - name: Delete infrastructure if failed due to lack of hosts
@@ -105,29 +96,21 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision_servers.outcome == 'failure' && contains(steps.provision_failure.messages, 'not enough hosts available') }}
 
       - name: Directly configure cluster
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
-          ansible-playbook -vv ansible/site.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+          ansible-playbook -v ansible/site.yml
+          ansible-playbook -v ansible/ci/check_slurm.yml
 
       - name: Run MPI-based tests
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
@@ -161,62 +144,62 @@ jobs:
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
       - name: Build packer images
+        id: packer_build
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          ansible-playbook ansible/adhoc/generate-passwords.yml
-          echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
           cd packer/
           PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
-          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          ../dev/output_manifest.py packer-manifest.json # Sets NEW_{COMPUTE,CONTROL,LOGIN}_IMAGE_ID outputs
 
-      - name: Test reimage of nodes
+      - name: Test reimage of login nodes (via rebuild adhoc)
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          ansible all -m wait_for_connection
-          ansible-playbook -vv ansible/ci/test_reimage.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
+          ansible-playbook -v --limit login ansible/adhoc/rebuild.yml -e rebuild_image=${{ steps.packer_build.outputs.NEW_LOGIN_IMAGE_ID }}
+          ansible login -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook -v ansible/ci/check_slurm.yml
+
+      - name: Test reimage of compute nodes (via slurm)
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible login -v -a "sudo scontrol reboot ASAP nextstate=RESUME reason='rebuild image:${{ steps.packer_build.outputs.NEW_COMPUTE_IMAGE_ID }}' ${TF_VAR_cluster_name}-compute-[0-3]"
+          ansible compute -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook -v ansible/ci/check_slurm.yml
+      
+      - name: Test reimage of control node (via rebuild adhoc)
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook -v --limit control ansible/adhoc/rebuild.yml -e rebuild_image=${{ steps.packer_build.outputs.NEW_CONTROL_IMAGE_ID }}
+          ansible control -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
+          ansible-playbook ansible/slurm.yml --tags openhpc # configures partitions
+          ansible-playbook ansible/monitoring.yml --tags prometheus # configures scrapes
+          ansible-playbook -v ansible/ci/check_slurm.yml
       
       - name: Check sacct state survived reimage
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Check MPI-based tests are shown in Grafana
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/check_grafana.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ success() || cancelled() }}
 
       - name: Delete images
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/delete_images.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True

ansible/adhoc/hpctests.yml

@@ -4,7 +4,7 @@
 
 ---
 
-- hosts: login[0] # TODO: might want to make which node is used selectable?
+- hosts: hpctests[0] # TODO: might want to make which node is used selectable?
   become: false
   gather_facts: false
   tasks:

ansible/bootstrap.yml

@@ -117,8 +117,6 @@
         async: "{{ 30 * 60 }}" # wait for up to 30 minutes
         poll: 15 # check every 15 seconds
         register: updates
-      - debug:
-          var: updates
       - name: Ensure update log directory on localhost exists
         file:
           path: "{{ update_log_path | dirname }}"

ansible/ci/check_slurm.yml

@@ -0,0 +1,25 @@
+- hosts: login:!builder # won't have a slurm control daemon when in build
+  become: no
+  gather_facts: false
+  tasks:
+    - name: Run sinfo
+      shell: 'sinfo --noheader --format="%N %P %a %l %D %t" | sort' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
+      register: sinfo
+      changed_when: false
+      until: "'boot' not in sinfo.stdout_lines"
+      retries: 5
+      delay: 10
+    - name: Check nodes have expected slurm state
+      assert:
+        that: sinfo.stdout_lines == expected_sinfo
+        fail_msg: |
+          sinfo output not as expected:
+          actual:
+          {{ sinfo.stdout_lines }}
+          expected:
+          {{ expected_sinfo }}
+          <end>
+      vars:
+        expected_sinfo:
+          - "{{ openhpc_cluster_name }}-compute-[0-1] small* up 60-00:00:00 2 idle"
+          - "{{ openhpc_cluster_name }}-compute-[2-3] extra up 60-00:00:00 2 idle"

ansible/ci/test_reimage.yml

@@ -35,6 +35,7 @@ Role Variables
 The following variables should not generally be changed:
 - `hpctests_pingmatrix_modules`: Optional. List of modules to load for pingmatrix test. Defaults are suitable for OpenHPC 2.x cluster using the required packages.
 - `hpctests_pingpong_modules`: As above but for pingpong test.
+- `hpctests_pingpong_plot`: Whether to plot pingpong results. Default `yes`.
 - `hpctests_hpl_modules`: As above but for hpl tests.
 - `hpctests_hpl_version`: Version of HPL
 

ansible/roles/hpctests/README.md

@@ -2,6 +2,7 @@
 hpctests_rootdir:
 hpctests_pingmatrix_modules: [gnu9 openmpi4]
 hpctests_pingpong_modules: [gnu9 openmpi4 imb]
+hpctests_pingpong_plot: yes
 hpctests_hpl_modules: [gnu9 openmpi4 openblas]
 hpctests_outdir: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/hpctests"
 hpctests_ucx_net_devices: all

ansible/roles/hpctests/defaults/main.yml

@@ -48,10 +48,11 @@
 
 - name: Plot image
   shell:
-    cmd: "python {{lookup('env', 'APPLIANCES_REPO_ROOT') }}/ansible/roles/hpctests/files/plot_imb_pingpong.py {{ _pingpong_local_output }}"
+    cmd: "python {{ role_path }}/files/plot_imb_pingpong.py {{ _pingpong_local_output }}"
     creates: "{{ _pingpong_local_output | dirname }}/latency.png"
   register: _pingpong_plot
   delegate_to: localhost
+  when: hpctests_pingpong_plot | bool
 
 - debug:
     msg: |
@@ -61,5 +62,7 @@
       Zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
       Max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
 
+      {% if hpctests_pingpong_plot %}
       See plot on localhost:
       {{ _pingpong_plot.stdout }}
+      {% endif %}

ansible/roles/hpctests/tasks/pingpong.yml

@@ -7,7 +7,7 @@
 
 - name: Select default partition if hpctests_partition not given
   set_fact:
-    hpctests_partition: "{{ _sinfo_partitions.stdout_lines  | select('contains', '*') | first | trim('*') }}"
+    hpctests_partition: "{{ (_sinfo_partitions.stdout_lines  | select('contains', '*') | first)[:-1] }}"
   when: hpctests_partition is not defined
 
 - name: Get info about compute nodes