Merge branch 'upstream' from v1.148 (incl. image update)

Author: sjpb

.github/workflows/fatimage.yml

@@ -3,24 +3,26 @@ name: Build fat image
 'on':
   workflow_dispatch:
     inputs:
-      use_RL9:
+      use_RL8:
         required: true
-        description: Include RL9 image build
+        description: Include RL8 image build
         type: boolean
         default: false
+concurrency:
+  group: ${{ github.ref }}-{{ matrix.os_version }} # to branch/PR + OS
+  cancel-in-progress: true
 jobs:
   openstack:
     name: openstack-imagebuild
     runs-on: ubuntu-20.04
-    concurrency: ${{ github.ref }}-{{ matrix.os_version }} # to branch/PR + OS
     strategy:
       matrix:
         os_version: [RL8, RL9]
-        rl9_selected:
-          - ${{ inputs.use_RL9 == true }} # only potentially true for workflow_dispatch
+        rl8_selected:
+          - ${{ inputs.use_RL8 == true }} # only potentially true for workflow_dispatch
         exclude:
-          - os_version: RL9
-            rl9_selected: false
+          - os_version: RL8
+            rl8_selected: false
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -61,18 +63,19 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
         env:
           PKR_VAR_os_version: ${{ matrix.os_version }}
 
-      - name: Get created image name from manifest
+      - name: Get created image names from manifest
         id: manifest
         run: |
           . venv/bin/activate
-          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
-          while ! openstack image show -f value -c name $IMAGE_ID; do
-            sleep 30
+          for IMAGE_ID in $(jq --raw-output '.builds[].artifact_id' packer/packer-manifest.json)
+          do
+            while ! openstack image show -f value -c name $IMAGE_ID; do
+              sleep 5
+            done
+            IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+            echo $IMAGE_NAME
           done
-          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
-          echo "IMAGE_ID=${IMAGE_ID}" >> "$GITHUB_OUTPUT"
-          echo "IMAGE_NAME=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"

.github/workflows/stackhpc.yml

@@ -3,9 +3,9 @@ name: Test deployment and reimage on OpenStack
 on:
   workflow_dispatch:
     inputs:
-      use_RL9:
+      use_RL8:
         required: true
-        description: Include RL9 tests
+        description: Include RL8 tests
         type: boolean
         default: false
   # push:
@@ -20,14 +20,17 @@ jobs:
     strategy:
       matrix:
         os_version: [RL8, RL9]
-        rl9_selected:
-          - ${{ inputs.use_RL9 == true }} # only potentially true for workflow_dispatch
-        rl9_branch:
-          - ${{ startsWith(github.head_ref, 'rl9') == true }} # only potentially for pull_request, always false on merge
+        rl8_selected:
+          - ${{ inputs.use_RL8 == true }} # only potentially true for workflow_dispatch
+        rl8_branch:
+          - ${{ startsWith(github.head_ref, 'rl8') == true }} # only potentially for pull_request, always false on merge
+        rl8_label:
+          - ${{ contains(github.event.pull_request.labels.*.name, 'RL8') }} # NB: needs a new commit if added after PR created
         exclude:
-          - os_version: RL9
-            rl9_selected: false
-            rl9_branch: false
+          - os_version: RL8
+            rl8_selected: false
+            rl8_branch: false
+            rl8_label: false
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack

ansible/.gitignore

@@ -52,4 +52,5 @@ roles/*
 !roles/image_build/**
 !roles/persist_hostkeys/
 !roles/persist_hostkeys/**
-!roles/requirements.yml
+!roles/ofed/
+!roles/ofed/**

ansible/bootstrap.yml

@@ -196,3 +196,11 @@
     - name: update facts
       setup:
       when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))
+
+- hosts: ofed
+  gather_facts: no
+  become: yes
+  tags: ofed
+  tasks:
+    - include_role:
+        name: ofed

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -116,6 +116,13 @@ data "openstack_networking_network_v2" "cluster_external_network" {
   name = "{{ cluster_external_network }}"
 }
 
+# Storage network 
+{% if cluster_storage_network is defined %}
+data "openstack_networking_network_v2" "cluster_storage" {
+  name = "{{ cluster_storage_network }}"
+}
+{% endif %}
+
 data "openstack_networking_subnet_ids_v2" "cluster_external_subnets" {
   network_id = "${data.openstack_networking_network_v2.cluster_external_network.id}"
 }
@@ -177,6 +184,11 @@ data "openstack_networking_subnet_v2" "cluster_subnet" {
 ##### Cluster ports
 #####
 
+###
+# Login node
+###
+
+# Primary network
 resource "openstack_networking_port_v2" "login" {
   name = "{{ cluster_name }}-login-0"
   network_id = "${data.openstack_networking_network_v2.cluster_network.id}"
@@ -193,14 +205,31 @@ resource "openstack_networking_port_v2" "login" {
 
   binding {
     vnic_type = "{{ cluster_vnic_type | default('normal') }}"
-    {% if cluster_vnic_profile is defined %}
-    profile = <<EOF
-    {{ cluster_vnic_profile | to_json }}
-    EOF
-    {% endif %}
   }
 }
 
+# Storage network
+{% if cluster_storage_network is defined %}
+resource "openstack_networking_port_v2" "login_storage" {
+  name           = "{{ cluster_name }}-login-storage-0"
+  network_id     = data.openstack_networking_network_v2.cluster_storage.id
+  admin_state_up = "true"
+
+  security_group_ids = [
+    "${openstack_networking_secgroup_v2.secgroup_slurm_cluster.id}",
+  ]
+
+  binding {
+    vnic_type = "{{ cluster_storage_vnic_type | default('normal') }}"
+  }
+}
+{% endif %}
+
+###
+# Control node
+###
+
+# Primary network
 resource "openstack_networking_port_v2" "control" {
   name = "{{ cluster_name }}-control-0"
   network_id = "${data.openstack_networking_network_v2.cluster_network.id}"
@@ -216,15 +245,32 @@ resource "openstack_networking_port_v2" "control" {
 
   binding {
     vnic_type = "{{ cluster_vnic_type | default('normal') }}"
-    {% if cluster_vnic_profile is defined %}
-    profile = <<EOF
-    {{ cluster_vnic_profile | to_json }}
-    EOF
-    {% endif %}
+
+  }
+}
+
+# Storage network
+{% if cluster_storage_network is defined %}
+resource "openstack_networking_port_v2" "control_storage" {
+  name           = "{{ cluster_name }}-control-storage-0"
+  network_id     = data.openstack_networking_network_v2.cluster_storage.id
+  admin_state_up = "true"
+
+  security_group_ids = [
+    "${openstack_networking_secgroup_v2.secgroup_slurm_cluster.id}"
+  ]
+
+  binding {
+    vnic_type = "{{ cluster_storage_vnic_type | default('normal') }}"
   }
 }
+{% endif %}
 
+###
+# Workers
+###
 {% for partition in openhpc_slurm_partitions %}
+# Primary network
 resource "openstack_networking_port_v2" "{{ partition.name }}" {
   count = {{ partition.count }}
   name = "{{ cluster_name }}-compute-{{ partition.name }}-${count.index}"
@@ -241,14 +287,27 @@ resource "openstack_networking_port_v2" "{{ partition.name }}" {
 
   binding {
     vnic_type = "{{ cluster_vnic_type | default('normal') }}"
-    {% if cluster_vnic_profile is defined %}
-    profile = <<EOF
-    {{ cluster_vnic_profile | to_json }}
-    EOF
-    {% endif %}
   }
 }
 
+# Storage network
+{% if cluster_storage_network is defined %}
+resource "openstack_networking_port_v2" "{{ partition.name }}_storage" {
+  count          = {{ partition.count }}
+  name           = "{{ cluster_name }}-compute-{{ partition.name }}-storage-${count.index}"
+  network_id     = data.openstack_networking_network_v2.cluster_storage.id
+  admin_state_up = "true"
+
+  security_group_ids = [
+    "${openstack_networking_secgroup_v2.secgroup_slurm_cluster.id}"
+  ]
+
+  binding {
+    vnic_type = "{{ cluster_storage_vnic_type | default('normal') }}"
+  }
+}
+{% endif %}
+
 {% endfor %}
 
 #####
@@ -274,9 +333,15 @@ resource "openstack_compute_instance_v2" "login" {
   {% endif %}
 
   network {
-    port = "${openstack_networking_port_v2.login.id}"
+    port = openstack_networking_port_v2.login.id
   }
 
+  {% if cluster_storage_network is defined %}
+  network {
+    port = openstack_networking_port_v2.login_storage.id
+  }
+  {% endif %}
+
   # root device:
   block_device {
       uuid = "{{ cluster_image }}"
@@ -317,9 +382,15 @@ resource "openstack_compute_instance_v2" "control" {
   {% endif %}
 
   network {
-    port = "${openstack_networking_port_v2.control.id}"
+    port = openstack_networking_port_v2.control.id
   }
 
+  {% if cluster_storage_network is defined %}
+  network {
+    port = openstack_networking_port_v2.control_storage.id
+  }
+  {% endif %}
+
   # root device:
   block_device {
       uuid = "{{ cluster_image }}"
@@ -393,6 +464,12 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
     port = openstack_networking_port_v2.{{ partition.name }}[count.index].id
   }
 
+  {% if cluster_storage_network is defined %}
+  network {
+    port = openstack_networking_port_v2.{{ partition.name }}_storage[count.index].id
+  }
+  {% endif %}
+
   # root device:
   block_device {
       uuid = "{{ cluster_image }}"

ansible/roles/freeipa/tasks/addhost.yml

@@ -4,7 +4,6 @@
   community.general.ipa_host:
     name: "{{ node_fqdn }}"
     ip_address: "{{ freeipa_client_ip }}"
-    ipa_host: "{{ groups['freeipa_server'].0 }}"
     ipa_pass: "{{ vault_freeipa_admin_password }}"
     ipa_user: admin
     state: present
@@ -19,12 +18,12 @@
   community.general.ipa_host:
     name: "{{ node_fqdn }}"
     ip_address: "{{ freeipa_client_ip }}"
-    ipa_host: "{{ groups['freeipa_server'].0 }}"
     ipa_pass: "{{ vault_freeipa_admin_password }}"
     ipa_user: admin
     random_password: true
     state: present
     validate_certs: false
+    ipa_timeout: 30
   delegate_to: "{{ groups['freeipa_server'].0 }}"
   when: "'sshpubkeyfp' not in _ipa_host_check.host"
   register: _ipa_host_add

ansible/roles/ofed/README.md

@@ -0,0 +1,21 @@
+# ofed
+
+This role installs Mellanox OFED:
+- It checks that the running kernel is the latest installed one, and errors if not.
+- Installation uses the `mlnxofedinstall` command, with support for the running kernel
+and (by default) without firmware updates.
+
+As OFED installation takes a long time generally this should only be used during image build,
+for example by setting:
+
+```
+environments/groups/<environment>/groups:
+[ofed:children]
+builder
+```
+
+# Role variables
+
+See `defaults/main.yml`
+
+Note ansible facts are required, unless setting `ofed_distro_version` and `ofed_arch` specifically.

ansible/roles/ofed/defaults/main.yml

@@ -0,0 +1,28 @@
+ofed_version: '24.04-0.6.6.0'  # LTS version 23.10-2.1.3.1 does not support RL9.4
+ofed_download_url: https://content.mellanox.com/ofed/MLNX_OFED-{{ ofed_version }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}.tgz
+ofed_distro: rhel # NB: not expected to work on other distros due to installation differences
+ofed_distro_version: "{{ ansible_distribution_version }}" # e.g. '8.9'
+ofed_arch: "{{ ansible_architecture }}"
+ofed_tmp_dir: /tmp
+ofed_update_firmware: false
+ofed_build_packages: # may require additional packages depending on ofed_package_selection
+  - autoconf
+  - automake
+  - gcc
+  - gcc-gfortran
+  - kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}
+  - kernel-rpm-macros
+  - libtool
+  - lsof
+  - patch
+  - pciutils
+  - perl
+  - rpm-build
+  - tcl
+  - tk
+ofed_build_rl8_packages:
+  - gdb-headless
+  - python36
+ofed_package_selection: # list of package selection flags for mlnxofedinstall script
+  - hpc
+  - with-nfsrdma