Merge branch 'stackhpc/2025.1' into feature/GT-1561-dell-sonic-2025.1

Author: dougszumski

.github/stackhpc-pull-request-path-filters.yml

@@ -11,8 +11,8 @@ aio: &aio
   - '.github/workflows/stackhpc-pull-request.yml'
   - 'etc/kayobe/*.yml'
   - 'etc/kayobe/ansible/tools/configure-aio-resources.yml'
-  - 'etc/kayobe/ansible/stackhpc-openstack-tests.yml'
   - 'etc/kayobe/ansible/tools/growroot.yml'
+  - 'etc/kayobe/ansible/tools/stackhpc-cloud-tests.yml'
   - 'etc/kayobe/ansible/requirements.yml'
   - 'etc/kayobe/ansible/scripts/aio-init.sh'
   - 'etc/kayobe/environments/ci-aio/**'

.github/workflows/ipa-image-build.yml

@@ -61,11 +61,6 @@ jobs:
         with:
           path: src/kayobe-config
 
-      - name: Output image tag of the builder
-        id: builder_image_tag
-        run: |
-          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
-
       - name: Determine OpenStack release
         id: openstack_release
         run: |

.github/workflows/multinode-inputs.py

@@ -2,6 +2,7 @@
 # The test scenario is randomly selected.
 # The inputs are printed to stdout in GitHub step output key=value format.
 
+import argparse
 from dataclasses import dataclass
 import random
 import typing as t
@@ -17,6 +18,7 @@ class OSRelease:
 @dataclass
 class OpenStackRelease:
     version: str
+    previous_version: str
     os_releases: t.List[OSRelease]
 
 
@@ -33,19 +35,41 @@ class Scenario:
 UBUNTU_NOBLE = OSRelease("ubuntu", "noble", "ubuntu")
 # NOTE(upgrade): Add supported releases here.
 OPENSTACK_RELEASES = [
-    OpenStackRelease("2023.1", [ROCKY_9, UBUNTU_JAMMY]),
-    OpenStackRelease("2024.1", [ROCKY_9, UBUNTU_JAMMY]),
-    OpenStackRelease("2025.1", [ROCKY_9, UBUNTU_NOBLE]),
+    OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2025.1", "2024.1", [ROCKY_9, UBUNTU_NOBLE]),
 ]
 NEUTRON_PLUGINS = ["ovs", "ovn"]
 VERSION_HIERARCHY = ["zed", "2023.1", "2024.1", "2025.1"]
 
 
 def main() -> None:
+
+    parser = argparse.ArgumentParser(
+        description='Randomly picks a multinode scenario to execute')
+    parser.add_argument(
+        '--output-summary', '-s',
+        type=argparse.FileType('w', encoding='UTF-8'),
+        default=None,
+        help="Write a markdown summary table of selected inputs to a file (use '-' to write to stdout)")
+    args = parser.parse_args()
+
     scenario = random_scenario()
-    inputs = generate_inputs(scenario)
+    inputs = {
+        "os_distribution": scenario.os_release.distribution,
+        "os_release": scenario.os_release.release,
+        "ssh_username": scenario.os_release.ssh_username,
+        "neutron_plugin": scenario.neutron_plugin,
+        "upgrade": scenario.upgrade,
+        "stackhpc_kayobe_config_version": get_branch(scenario.openstack_release.version),
+        "stackhpc_kayobe_config_previous_version": get_branch(scenario.openstack_release.previous_version),
+        "terraform_kayobe_multinode_version": get_tkm_version(scenario.openstack_release.version),
+        "terraform_kayobe_multinode_previous_version": get_tkm_version(scenario.openstack_release.previous_version),
+    }
     for name, value in inputs.items():
         write_output(name, value)
+    if args.output_summary:
+        write_summary(inputs, args.output_summary)
 
 
 def random_scenario() -> Scenario:
@@ -56,43 +80,30 @@ def random_scenario() -> Scenario:
     return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
 
 
-def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
-    branch = get_branch(scenario.openstack_release.version)
-    previous_branch = get_branch(
-        VERSION_HIERARCHY[
-            VERSION_HIERARCHY.index(scenario.openstack_release.version) - 1
-        ]
-    )
-    terraform_kayobe_multinode_version = get_tkm_version(
-        scenario.openstack_release.version
-    )
-    inputs = {
-        "os_distribution": scenario.os_release.distribution,
-        "os_release": scenario.os_release.release,
-        "ssh_username": scenario.os_release.ssh_username,
-        "neutron_plugin": scenario.neutron_plugin,
-        "upgrade": scenario.upgrade,
-        "stackhpc_kayobe_config_version": branch,
-        "stackhpc_kayobe_config_previous_version": previous_branch,
-        "terraform_kayobe_multinode_version": terraform_kayobe_multinode_version,
-    }
-    return inputs
-
-
 def get_branch(version: str) -> str:
     return f"stackhpc/{version}"
 
 
 def get_tkm_version(version: str) -> str:
-    if version in ["zed", "2023.1"]:
+    if version == "zed":
         return "ea61ea1730e179e05e8f0e58b759267664c555e7"
-    else:
+    elif version == "2025.1":
         return "main"
+    else:
+        return get_branch(version)
 
 
 def write_output(name: str, value: str) -> None:
     print(f"{name}={value}")
 
 
+def write_summary(inputs: dict, output: t.TextIO):
+    print(
+        '| Input  | Value |\n'
+        '| -----: | :---- |', file=output)
+    for key, value in inputs.items():
+        print(f'| **{key}** | `{value}`  |', file=output)
+
+
 if __name__ == "__main__":
     main()

.github/workflows/overcloud-host-image-build.yml

@@ -72,11 +72,6 @@ jobs:
         with:
           path: src/kayobe-config
 
-      - name: Output image tag of the builder
-        id: builder_image_tag
-        run: |
-          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
-
       - name: Determine OpenStack release
         id: openstack_release
         run: |
@@ -253,22 +248,6 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: steps.build_rocky_9.outcome == 'failure'
 
-      - name: Upload Rocky Linux 9 overcloud host image to Ark
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
-          -e artifact_path=/opt/kayobe/images/overcloud-rocky-9 \
-          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
-          -e artifact_type="kayobe-images" \
-          -e file_regex="*.qcow2" \
-          -e os_distribution="rocky" \
-          -e os_release="9"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
-
       - name: Upload Rocky Linux 9 overcloud host image to current Dev Cloud (SMS/Leafcloud)
         run: |
           source venvs/kayobe/bin/activate &&
@@ -297,6 +276,22 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }}
         if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
 
+      - name: Upload Rocky Linux 9 overcloud host image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/overcloud-rocky-9 \
+          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          -e artifact_type="kayobe-images" \
+          -e file_regex="*.qcow2" \
+          -e os_distribution="rocky" \
+          -e os_release="9"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
+
       - name: Build an Ubuntu Noble 24.04 overcloud host image
         id: build_ubuntu_noble
         continue-on-error: true
@@ -321,22 +316,6 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: steps.build_ubuntu_noble.outcome == 'failure'
 
-      - name: Upload Ubuntu Noble 24.04 overcloud host image to Ark
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
-          -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-noble \
-          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
-          -e artifact_type="kayobe-images" \
-          -e file_regex="*.qcow2" \
-          -e os_distribution="ubuntu" \
-          -e os_release="noble"
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success'
-
       - name: Upload Ubuntu Noble overcloud host image to current Dev Cloud (SMS/Leafcloud)
         run: |
           source venvs/kayobe/bin/activate &&
@@ -365,6 +344,22 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }}
         if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success'
 
+      - name: Upload Ubuntu Noble 24.04 overcloud host image to Ark
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp/pulp-artifact-upload.yml \
+          -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-noble \
+          -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
+          -e artifact_type="kayobe-images" \
+          -e file_regex="*.qcow2" \
+          -e os_distribution="ubuntu" \
+          -e os_release="noble"
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success'
+
       - name: Copy logs back
         continue-on-error: true
         run: |

.github/workflows/package-build-ofed.yml

@@ -97,10 +97,12 @@ jobs:
           aio_vm_network = "stackhpc-ci"
           aio_vm_subnet = "stackhpc-ci"
           aio_vm_interface = "ens3"
+          aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
         working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
         env:
           VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
+          VM_TAGS: '["skc-ci-doca-builder", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
         run: terraform plan
@@ -233,7 +235,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/tools/push-ofed.yml \
           -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}

.github/workflows/stackhpc-ci-cleanup.yml

@@ -60,6 +60,26 @@ jobs:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
+      - name: Clean up DOCA builder instances over 5 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -5hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-ci-doca-builder --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
       - name: Clean up host image builder instances over 5 hours old
         run: |
           result=0

.github/workflows/stackhpc-container-image-build.yml

@@ -155,7 +155,7 @@ jobs:
 
       - name: Install Trivy
         run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.66.0
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.68.2
 
       - name: Install yq
         run: |
@@ -175,7 +175,12 @@ jobs:
       # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          sudo pip install docker 'requests<2.32.0'
+          . /etc/os-release
+          if [[ $VERSION_ID == 22.04 ]]; then
+              sudo pip install docker 'requests<2.32.0'
+          else
+              sudo pip install docker 'requests<2.32.0' --break-system-packages
+          fi
 
       - name: Get Kolla tag
         id: write-kolla-tag

.github/workflows/stackhpc-multinode-periodic.yml

@@ -20,14 +20,16 @@ jobs:
       stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }}
       stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
       terraform_kayobe_multinode_version: ${{ steps.generate-inputs.outputs.terraform_kayobe_multinode_version }}
+      terraform_kayobe_multinode_previous_version: ${{ steps.generate-inputs.outputs.terraform_kayobe_multinode_previous_version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Generate inputs for multinode workflow
         id: generate-inputs
         run: |
-          python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT
+          python3 .github/workflows/multinode-inputs.py -s "$GITHUB_STEP_SUMMARY" \
+            | tee -a "$GITHUB_OUTPUT"
 
       - name: Display generated inputs
         run: |
@@ -36,7 +38,7 @@ jobs:
     name: Multinode periodic
     needs:
       - generate-inputs
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.4.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.5.0
     with:
       multinode_name: mn-periodic-${{ github.run_id }}
       os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}
@@ -47,6 +49,7 @@ jobs:
       stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }}
       stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
       terraform_kayobe_multinode_version: ${{ needs.generate-inputs.outputs.terraform_kayobe_multinode_version }}
+      terraform_kayobe_multinode_previous_version: ${{ needs.generate-inputs.outputs.terraform_kayobe_multinode_previous_version }}
       enable_slack_alert: true
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'