DOCA-ofed: stackhpc_doca_kernel_version has the kernel module version to install

It must be bumped after a new pre-compiled kernel module version has been built.

Author: elelaysh

etc/kayobe/ansible/maintenance/rocky-97-ofed-upgrade.yml

@@ -15,7 +15,9 @@
   gather_facts: true
   tags: pre
   vars:
-    doca_kernel_version: 5.14.0.611.13.1.el9.7.x86.64
+    # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
+    # but not 5.14.0-611.20.1.
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version[stackhpc_pulp_repo_rocky_9_minor_version] }}"
   tasks:
     - name: Assert that hosts are running Rocky Linux 9.6
       ansible.builtin.assert:
@@ -35,11 +37,11 @@
         update_cache: true
       become: true
 
-    # This is required by doca-kernel-xxx, and comes from the doca repository.
+    # This is required by mlnx-ofa_kernel, and comes from the doca repository.
     # It is already present when doca-ofed is installed, but will be upgraded.
-    - name: Ensure mlnx-ofa_kernel is installed
+    - name: Ensure mlnx-tools is installed
       ansible.builtin.dnf:
-        name: mlnx-ofa_kernel
+        name: mlnx-tools
         state: latest
         update_cache: true
       become: true

etc/kayobe/ansible/tools/install-doca.yml

@@ -3,27 +3,50 @@
   become: true
   hosts: mlnx
   gather_facts: true
+  vars:
+    # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
+    # but not 5.14.0-611.20.1
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version[stackhpc_pulp_repo_rocky_9_minor_version] }}"
   tasks:
-    - name: Get running kernel
-      ansible.builtin.command:
-        cmd: "uname -r"
-      register: kernel
-      check_mode: false
-
     - name: Install kernel repo
       ansible.builtin.dnf:
         name: doca-kernel-repo
         state: latest
         update_cache: true
 
+    # not the same as doca_kernel_version: some dots changed to underscore or dash
+    - name: Discover kernel repo filename
+      ansible.builtin.shell: |
+        set -o pipefail
+        rpm -ql doca-kernel-repo | grep /etc/yum.repos.d/
+      register: kernel_repo_filename
+      changed_when: false
+
     - name: Ensure correct priority for DOCA modules
       ansible.builtin.lineinfile:
         line: "priority=-2"
         insertafter: EOF
-        path: "/etc/yum.repos.d/doca-kernel-{{ kernel.stdout }}.repo"
+        path: "{{ kernel_repo_filename.stdout }}"
+
+    # This is required by mlnx-ofa_kernel, and comes from the doca repository.
+    # It is already present when doca-ofed is installed, but will be upgraded.
+    - name: Ensure mlnx-tools is installed
+      ansible.builtin.dnf:
+        name: mlnx-tools
+        state: latest
+        update_cache: true
+      become: true
+
+    - name: Ensure appropriate doca-kernel is installed
+      ansible.builtin.dnf:
+        name: "doca-kernel-{{ doca_kernel_version }}"
+        state: latest
+        disablerepo: doca
+      become: true
 
-    - name: Install DOCA OFED
+    - name: Ensure DOCA OFED userspace is installed
       ansible.builtin.dnf:
-        name: doca-ofed
+        name:
+          - doca-ofed-userspace
         state: latest
         update_cache: true

etc/kayobe/ofed.yml

@@ -8,6 +8,11 @@ stackhpc_pulp_doca_version_matrix:
     "7": 3.2.1
 stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] | default('2.9.1') }}"
 
+# Available and tested versions of the pre-compiled doca-ofed kernel modules
+stackhpc_doca_kernel_version:
+    "6": 5.14.0.570.21.1.el9.6.x86.64
+    "7": 5.14.0.611.13.1.el9.7.x86.64
+
 ###############################################################################
 # Pulp configuration for DOCA OFED