fix: octavia-api broken in multinode OVS

.github/workflows/stackhpc-all-in-one.yml

@@ -314,6 +314,26 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Pull container images
+        run: |
+          # Try up to 3 times to pull container images.
+          for attempt in $(seq 3); do
+              docker run -t --rm \
+                -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+                -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+                ${{ steps.kayobe_image.outputs.kayobe_image }} \
+                /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh
+              if [ "$?" = "0" ]; then
+                  echo "Pulled container images on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to pull container images on attempt $attempt"
+          done
+          echo "Failed to pull container imagers after $attempt attempts"
+          exit 1
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Service deploy
         run: |
           docker run -t --rm \

.github/workflows/stackhpc-container-image-build.yml

@@ -82,14 +82,14 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT"
 
       # Generate a tag to apply to all built container images.
       # Without this, each kayobe * container image build command would use a different tag.
       - name: Generate container datetime tag
         id: datetime_tag
         run: |
-          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT"
 
       # Dynamically define job matrix.
       # We need a separate matrix entry for each distribution, when the relevant input is true.
@@ -110,7 +110,7 @@ jobs:
           # remove trailing comma
           output="${output%,}"
           output+="]}"
-          echo "matrix=$output" >> $GITHUB_OUTPUT
+          echo "matrix=$output" | tee -a "$GITHUB_OUTPUT"
 
       - name: Display container datetime tag
         run: |
@@ -184,7 +184,7 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT"
 
       - name: Configure localhost as a seed
         run: |
@@ -250,7 +250,7 @@ jobs:
         if: inputs.seed && matrix.distro.arch == 'amd64'
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images"
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
@@ -260,6 +260,7 @@ jobs:
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
+        if: ${{ !cancelled() }}
 
       - name: Fail if any images have critical vulnerabilities
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi

doc/source/operations/octavia.rst

@@ -87,8 +87,8 @@ Copy the relevant certificates into your kayobe-config:
    cd ${KAYOBE_CONFIG_PATH}/environments/$KAYOBE_ENVIRONMENT/kolla/config/octavia
    cp $KOLLA_CONFIG_PATH/octavia-certificates/client_ca/client_ca.cert.pem .
    cp $KOLLA_CONFIG_PATH/octavia-certificates/client_ca/client.cert-and-key.pem .
-   cp $KOLLA_CONFIG_PATH/octavia-certificates/client_ca/server_ca.cert.pem .
-   cp $KOLLA_CONFIG_PATH/octavia-certificates/client_ca/server_ca.key.pem .
+   cp $KOLLA_CONFIG_PATH/octavia-certificates/server_ca/server_ca.cert.pem .
+   cp $KOLLA_CONFIG_PATH/octavia-certificates/server_ca/server_ca.key.pem .
 
 Encrypt any files containing the keys:
 
@@ -232,7 +232,7 @@ Access the database from a controller:
 .. code-block:: console
 
   docker exec -it mariadb bash
-  mysql -u root -p  octavia
+  mysql -u octavia -p  octavia
   # Enter the database password when prompted.
 
 List the load balancers to find the ID of the broken one(s):

etc/kayobe/environments/ci-multinode/controllers.yml

@@ -5,7 +5,6 @@ controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' el
 controller_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"
 
-
 ###############################################################################
 # Controller node firewalld configuration.
 
@@ -28,3 +27,7 @@ controller_firewalld_default_zone: trusted
 # - permanent: true
 # - state: enabled
 controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}"
+
+# List of extra networks to which controller nodes are attached.
+controller_extra_network_interfaces:
+  - octavia

etc/kayobe/environments/ci-multinode/globals.yml

@@ -63,6 +63,9 @@ stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id"
 # Enable rebooting to update SELinux state
 selinux_do_reboot: true
 
+# For ansible/maintenance/octavia-amphora-image-register.yml
+openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}"
+
 ###############################################################################
 # Extra vars.
 

etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces

@@ -16,6 +16,8 @@ public_routes:
   - cidr: "{{ external_cidr }}"
     gateway: "{{ public_net_name | net_ip( groups['seed'][0] ) }}"
 
+octavia_interface: "{{ vxlan_interfaces[0].device}}.{{ octavia_vlan }}"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

etc/kayobe/environments/ci-multinode/kolla/globals.yml

@@ -47,11 +47,6 @@ neutron_dns_domain: "{{ root_domain }}."
 # OpenSearch memory tuning
 opensearch_heap_size: 1g
 
-# Octavia load balancer configuration
-octavia_auto_configure: "no"
-octavia_provider_drivers: "ovn:OVN provider"
-octavia_provider_agents: "ovn"
-
 # Manila CephFS configuration
 manila_cephfs_filesystem_name: manila-cephfs
 
@@ -70,3 +65,21 @@ external_api_firewalld_zone: "{{ public_net_name | net_zone }}"
 
 # Test cinder cluster
 cinder_cluster_name: "stackhpc_test_cinder_cluster"
+
+# Octavia load balancer configuration
+octavia_network_interface: "{{ hostvars[groups['controllers'] | first].octavia_interface }}"
+
+octavia_amp_network:
+  name: lb-mgmt-net
+  provider_network_type: vlan
+  provider_physical_network: "physnet1"
+  provider_segmentation_id: 999
+  external: false
+  shared: false
+  subnet:
+    name: lb-mgmt-subnet
+    cidr: "192.168.34.0/24"
+    allocation_pool_start: "192.168.34.10"
+    allocation_pool_end: "192.168.34.254"
+    no_gateway_ip: yes
+    enable_dhcp: yes

etc/kayobe/environments/ci-multinode/networks.yml

@@ -131,6 +131,13 @@ provision_oc_allocation_pool_end: 192.168.33.254
 provision_oc_vlan: 107
 provision_oc_zone: "provision_oc"
 
+octavia_cidr: 192.168.34.0/24
+octavia_vlan: 999
+octavia_allocation_pool_start: 192.168.34.3
+octavia_allocation_pool_end: 192.168.34.9
+octavia_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}"
+octavia_zone: octavia
+
 ###############################################################################
 # Network virtual patch link configuration.
 

etc/kayobe/environments/ci-multinode/stackhpc-ci.yml

@@ -61,3 +61,6 @@ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?:
 
 stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}"
 stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}"
+
+stackhpc_release_pulp_username: "{{ stackhpc_repo_mirror_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_repo_mirror_password }}"

etc/kayobe/kolla-image-tags.yml

@@ -4,28 +4,19 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky-9: 2025.1-rocky-9-20250730T105631
+    rocky-9: 2025.1-rocky-9-20260123T132550
     ubuntu-noble: 2025.1-ubuntu-noble-20250730T105631
   iscsid:
-    rocky-9: 2025.1-rocky-9-20251013T135156
     ubuntu-noble: 2025.1-ubuntu-noble-20251013T135156
   keystone:
-    rocky-9: 2025.1-rocky-9-20251031T094539
     ubuntu-noble: 2025.1-ubuntu-noble-20251031T094539
   kolla_toolbox:
-    rocky-9: 2025.1-rocky-9-20250905T151507
     ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507
   magnum:
-    rocky-9: 2025.1-rocky-9-20251016T073648
     ubuntu-noble: 2025.1-ubuntu-noble-20251016T073648
   neutron:
-    rocky-9: 2025.1-rocky-9-20260116T134029
     ubuntu-noble: 2025.1-ubuntu-noble-20260116T134029
-  ovn:
-    rocky-9: 2025.1-rocky-9-20250915T100822
   prometheus_libvirt_exporter:
-    rocky-9: 2025.1-rocky-9-20250822T151934
     ubuntu-noble: 2025.1-ubuntu-noble-20250822T151934
   rabbitmq:
-    rocky-9: 2025.1-rocky-9-20250905T151507
     ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507

etc/kayobe/kolla.yml

@@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}"
 # Path in which to generate kolla configuration.
 #kolla_build_config_path:
 
+# View debug logs, like source downloads, during image build.
+kolla_build_extra_config: |
+  [DEFAULT]
+  debug = True
+
 ###############################################################################
 # Kolla-ansible installation.
 

etc/kayobe/kolla/kolla-build.conf

@@ -29,3 +29,8 @@ sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,
 [magnum-conductor-plugin-helm]
 version = v3.18.2
 sha256 = amd64:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce,arm64:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d
+
+# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available
+[prometheus-cadvisor]
+version = 0.54.1
+sha256 = amd64:21be8d2797433048474e676d37c215c28fb171509448ef9b1c4648a564e39595,arm64:21f7bac786f6c53a8091964b4d3ff2486a0c460e5a410000b59a9a565b4183a9

etc/kayobe/pulp-host-image-versions.yml

@@ -1,5 +1,5 @@
 ---
 # Overcloud host image versioning tags
 # These images must be in SMS, since they are used by our AIO CI runners
-stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260113T105634"
+stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260119T110913"
 stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250930T144255"

etc/kayobe/trivy/allowed-vulnerabilities.yml

@@ -20,7 +20,10 @@ influxdb_allowed_vulnerabilities:
   - CVE-2024-45337
 magnum_conductor_allowed_vulnerabilities:
   - CVE-2024-45337
+opensearch_dashboards_allowed_vulnerabilities:
+  - CVE-2025-68428
 prometheus_blackbox_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
 prometheus_memcached_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
@@ -31,7 +34,10 @@ prometheus_elasticsearch_exporter_allowed_vulnerabilities:
 prometheus_node_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
 prometheus_openstack_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
+prometheus_ovn_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
 prometheus_libvirt_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
 prometheus_cadvisor_allowed_vulnerabilities:

releasenotes/notes/rl9.7-c8d033f11097b76b.yaml

@@ -1,7 +1,8 @@
 ---
 features:
   - |
-    Added support for Rocky Linux 9.7.
+    Added support for Rocky Linux 9.7, including host packages and a full
+    container image refresh.
   - |
     9.7 is now the default release for Rocky Linux.