NO-ISSUE: Test and script fixes found during 5.0.0-rc.0 testing

scripts/microshift-cleanup-data.sh

@@ -107,6 +107,15 @@ function clean_processes() {
         for pname in conmon pause ovn-controller ovn-northd ; do
             pkill -9 --exact ${pname} || true
         done
+        # Stop OVS so the stale datapath/flow state held by ovs-vswitchd is
+        # cleared (it is left inconsistent once ovsdb-server is stopped above),
+        # otherwise OVN cannot reinitialize on the next MicroShift start. The
+        # next start brings openvswitch back up via microshift's Wants and
+        # microshift-ovs-init's Requires, giving OVN a clean slate. Unlike the
+        # best-effort kills above, a real stop failure here has a correctness
+        # impact, so surface it instead of discarding stderr.
+        systemctl stop openvswitch.service \
+            || echo "WARN: failed to stop openvswitch.service; stale OVS flow state may persist" >&2
     fi
 }
 

test/bin/ci_phase_boot_and_test.sh

@@ -116,9 +116,12 @@ elif [[ "${SCENARIO_SOURCES}" =~ .*releases.* ]]; then
     # hypervisor only ever holds the still-running scenarios' VMs.
     jobs_arg="-j 20"
     scenario_action="create-run-shutdown"
-
-    # Give release scenarios a longer per-scenario execution timeout since
-    # many now run both the standard1 and standard2 suites.
+    # Release scenarios run upgrade paths with LVMS workloads followed by
+    # full standard suites (many now run both standard1 and standard2), which
+    # need more time than the default 30m robot timeout and 600s greenboot
+    # healthcheck — especially under I/O contention when many VMs boot and
+    # pull images in parallel.
+    export GREENBOOT_TIMEOUT=1200
     export TEST_EXECUTION_TIMEOUT=60m
 fi
 

test/resources/journalctl.py

@@ -1,6 +1,7 @@
 from robot.libraries.BuiltIn import BuiltIn
 
 import libostree
+import re
 import time
 
 _log = BuiltIn().log
@@ -27,28 +28,55 @@ def get_journal_cursor(unit="microshift") -> str:
     return cursor
 
 
-def get_log_output_with_pattern(cursor: str, pattern: str, unit="microshift") -> tuple[str, int]:
+def get_log_output_with_pattern(cursor: str, pattern: str, unit="microshift", exceptions=None) -> tuple[str, int]:
     """
     Get the logs since the cursor matching the pattern and return the log content and exit code.
     Optional argument `unit` may be used to specify a systemd unit other than microshift,
     for example microshift-observability.service.
     Note that this function ignores case when matching the pattern.
+
+    Optional argument `exceptions` is a list of regular expressions describing
+    known-benign log lines to ignore. Matching lines are dropped from the
+    result; if every matched line is an exception, the return code is
+    downgraded to 1 (no relevant match) so callers treat it as "not found".
     """
     stdout, rc = libostree.remote_sudo_rc(
         f"journalctl -u {unit} --cursor='{cursor}' --no-pager --case-sensitive=false --grep '{pattern}'"
     )
+    if exceptions and rc == 0:
+        matched = [line for line in stdout.splitlines() if line.strip()]
+        remaining = [
+            line for line in matched
+            if not any(re.search(exc, line, re.IGNORECASE) for exc in exceptions)
+        ]
+        dropped = len(matched) - len(remaining)
+        if dropped:
+            # Make the suppression visible: a benign exception that turns
+            # persistent (e.g. a ServiceAccount that never lands) would
+            # otherwise be masked silently.
+            BuiltIn().log(
+                f"Ignored {dropped} known-benign '{pattern}' line(s) via exceptions", "WARN"
+            )
+        if not remaining:
+            # All matches were known-benign exceptions; report no relevant match.
+            rc = 1
+        stdout = "\n".join(remaining)
     BuiltIn().log(f"log lines matching '{pattern}':\n{stdout}")
     return stdout, rc
 
 
-def pattern_should_not_appear_in_log_output(cursor, pattern, unit="microshift", retries=30, wait=10):
-    """Get the logs since the cursor and verify that the pattern does not appear."""
+def pattern_should_not_appear_in_log_output(cursor, pattern, unit="microshift", retries=30, wait=10, exceptions=None):
+    """Get the logs since the cursor and verify that the pattern does not appear.
+
+    Optional argument `exceptions` is a list of regular expressions describing
+    known-benign log lines that should not count as a match.
+    """
     # The grep argument causes journalctl to exit with an error if the
     # pattern is not found, therefore we want the return code to be 1,
     # indicating that there was no match.
 
     for attempt in range(1, retries + 2):
-        stdout, rc = get_log_output_with_pattern(cursor, pattern, unit)
+        stdout, rc = get_log_output_with_pattern(cursor, pattern, unit, exceptions)
         if rc == 1 or attempt > retries:
             BuiltIn().should_be_equal_as_integers(rc, 1)
             return

test/resources/kubeconfig.resource

@@ -92,6 +92,21 @@ Create Random Namespace
     RETURN    test-${rand}
 
 Remove Namespace
-    [Documentation]    Removes the given namespace.
+    [Documentation]    Removes the given namespace and waits until it is actually
+    ...    gone so a subsequent test can safely reuse the name. The delete
+    ...    tolerates failure (allow_fail) so a slow deletion killed at the Run
+    ...    With Kubeconfig process timeout under load does not fail the teardown.
     [Arguments]    ${ns}
-    Run With Kubeconfig    oc delete namespace ${ns}
+    Run With Kubeconfig    oc delete namespace ${ns}    allow_fail=True
+    Wait Until Keyword Succeeds    5m    5s    Namespace Should Not Exist    ${ns}
+
+Namespace Should Not Exist
+    [Documentation]    Fails unless the namespace is reported NotFound, so that a
+    ...    transient API error (which also exits non-zero) is not mistaken for a
+    ...    successful deletion.
+    [Arguments]    ${ns}
+    ${stdout}    ${rc}=    Run With Kubeconfig    oc get namespace ${ns}
+    ...    allow_fail=True    return_rc=True
+    Should Not Be Equal As Integers    ${rc}    0    Namespace ${ns} still exists
+    Should Contain    ${stdout}    NotFound
+    ...    msg=oc get namespace ${ns} failed for a reason other than NotFound: ${stdout}

test/resources/systemd.resource

@@ -85,3 +85,27 @@ Systemctl Daemon Reload
     ...    sudo=True    return_stdout=True    return_stderr=True    return_rc=True
     Log Many    ${stdout}    ${stderr}
     Should Be Equal As Integers    0    ${rc}
+
+Disable Journal Rate Limiting
+    [Documentation]    Disable journald rate limiting by writing a drop-in that
+    ...    sets RateLimitBurst=0, then restarting the journal service.
+    ${stdout}    ${stderr}    ${rc}=    Execute Command
+    ...    bash -c "mkdir -p /etc/systemd/journald.conf.d && printf '[Journal]\nRateLimitBurst=0\n' > /etc/systemd/journald.conf.d/disable-ratelimit.conf && systemctl restart systemd-journald"
+    ...    sudo=True
+    ...    return_stdout=True
+    ...    return_stderr=True
+    ...    return_rc=True
+    Log Many    ${stdout}    ${stderr}
+    Should Be Equal As Integers    0    ${rc}
+
+Enable Journal Rate Limiting
+    [Documentation]    Re-enable default journald rate limiting by removing the
+    ...    drop-in created by Disable Journal Rate Limiting.
+    ${stdout}    ${stderr}    ${rc}=    Execute Command
+    ...    bash -c "rm -f /etc/systemd/journald.conf.d/disable-ratelimit.conf && systemctl restart systemd-journald"
+    ...    sudo=True
+    ...    return_stdout=True
+    ...    return_stderr=True
+    ...    return_rc=True
+    Log Many    ${stdout}    ${stderr}
+    Should Be Equal As Integers    0    ${rc}

test/scenarios-bootc/el10/releases/el102-lrel@ginkgo-tests.sh

@@ -11,7 +11,7 @@ scenario_create_vms() {
     fi
 
     prepare_kickstart host1 kickstart-bootc.ks.template "${start_image}"
-    launch_vm rhel102-bootc --vm_vcpus 4
+    launch_vm rhel102-bootc --vm_disksize 30 --vm_vcpus 4
 }
 
 scenario_remove_vms() {

test/scenarios-bootc/el10/releases/el102-lrel@optional-sigstore.sh

@@ -2,15 +2,6 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-# Each optional suite restarts MicroShift with its own kustomizePaths config,
-# adding ~10 minutes of restart overhead to the total execution time.
-# shellcheck disable=SC2034  # used elsewhere
-TEST_EXECUTION_TIMEOUT=60m
-
-# shellcheck disable=SC2034  # used elsewhere
-# Increase greenboot timeout for optional packages (more services to start)
-GREENBOOT_TIMEOUT=1200
-
 # Enable container signature verification for current release images,
 # including the optional components.
 # These are ec / rc / z-stream, thus must all to be signed.

test/scenarios-bootc/el10/releases/el102-lrel@optional.sh

@@ -2,15 +2,6 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-# Each optional suite restarts MicroShift with its own kustomizePaths config,
-# adding ~10 minutes of restart overhead to the total execution time.
-# shellcheck disable=SC2034  # used elsewhere
-TEST_EXECUTION_TIMEOUT=60m
-
-# shellcheck disable=SC2034  # used elsewhere
-# Increase greenboot timeout for optional packages (more services to start)
-GREENBOOT_TIMEOUT=1200
-
 # Redefine network-related settings to use the dedicated network bridge
 VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")"
 # shellcheck disable=SC2034  # used elsewhere

test/scenarios-bootc/el10/releases/el102-y1@el102-lrel@lvms-standard.sh

@@ -6,9 +6,6 @@
 # ensure MicroShift is upgraded before running validation tests
 export TEST_RANDOMIZATION=none
 
-# Add extra timeout because it runs both standard1 and standard2 suites.
-export TEST_EXECUTION_TIMEOUT=60m
-
 start_image="rhel102-bootc-brew-y1-with-optional"
 dest_image="rhel102-bootc-brew-lrel-optional"
 

test/scenarios-bootc/el10/releases/el102-y1@el102-lrel@rpm-standard.sh

@@ -13,8 +13,6 @@ export SKIP_GREENBOOT=true
 # did not want to spend the resources on a new VM.
 export TEST_RANDOMIZATION=none
 
-export TEST_EXECUTION_TIMEOUT=60m
-
 scenario_create_vms() {
     exit_if_brew_rpms_not_found
 

test/scenarios-bootc/el10/releases/el102@rpm-standard.sh

@@ -13,9 +13,6 @@ export SKIP_GREENBOOT=true
 # did not want to spend the resources on a new VM.
 export TEST_RANDOMIZATION=none
 
-# Add extra timeout because it run both standard1 and standard2 suites.
-export TEST_EXECUTION_TIMEOUT=60m
-
 scenario_create_vms() {
     exit_if_brew_rpms_not_found
 

test/scenarios-bootc/el10/releases/el98-y1@el102-lrel@lvms-standard.sh

@@ -6,9 +6,6 @@
 # ensure MicroShift is upgraded before running validation tests
 export TEST_RANDOMIZATION=none
 
-# Add extra timeout because it runs both standard1 and standard2 suites.
-export TEST_EXECUTION_TIMEOUT=60m
-
 start_image="rhel98-bootc-brew-y1-with-optional"
 dest_image="rhel102-bootc-brew-lrel-optional"
 

test/scenarios-bootc/el10/releases/el98-y2@el102-lrel@lvms-standard.sh

@@ -6,9 +6,6 @@
 # ensure MicroShift is upgraded before running validation tests
 export TEST_RANDOMIZATION=none
 
-# Add extra timeout because it runs both standard1 and standard2 suites.
-export TEST_EXECUTION_TIMEOUT=60m
-
 start_image="rhel98-bootc-brew-y2-with-optional"
 dest_image="rhel102-bootc-brew-lrel-optional"
 

test/scenarios-bootc/el9/releases/el98-lrel@optional-sigstore.sh

@@ -2,15 +2,6 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-# Each optional suite restarts MicroShift with its own kustomizePaths config,
-# adding ~10 minutes of restart overhead to the total execution time.
-# shellcheck disable=SC2034  # used elsewhere
-TEST_EXECUTION_TIMEOUT=60m
-
-# shellcheck disable=SC2034  # used elsewhere
-# Increase greenboot timeout for optional packages (more services to start)
-GREENBOOT_TIMEOUT=1200
-
 # Enable container signature verification for current release images,
 # including the optional components.
 # These are ec / rc / z-stream, thus must all to be signed.

test/scenarios-bootc/el9/releases/el98-lrel@optional.sh

@@ -2,15 +2,6 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-# Each optional suite restarts MicroShift with its own kustomizePaths config,
-# adding ~10 minutes of restart overhead to the total execution time.
-# shellcheck disable=SC2034  # used elsewhere
-TEST_EXECUTION_TIMEOUT=60m
-
-# shellcheck disable=SC2034  # used elsewhere
-# Increase greenboot timeout for optional packages (more services to start)
-GREENBOOT_TIMEOUT=1200
-
 # Redefine network-related settings to use the dedicated network bridge
 VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")"
 # shellcheck disable=SC2034  # used elsewhere

test/suites/configuration2/logging.robot

@@ -4,6 +4,7 @@ Documentation       Tests for case-insensitive log level parsing
 Resource            ../../resources/common.resource
 Resource            ../../resources/microshift-config.resource
 Resource            ../../resources/microshift-process.resource
+Resource            ../../resources/systemd.resource
 Library             ../../resources/journalctl.py
 
 Suite Setup         Setup
@@ -29,10 +30,12 @@ Setup
     Check Required Env Variables
     Login MicroShift Host
     Setup Kubeconfig
+    Disable Journal Rate Limiting
 
 Teardown
     [Documentation]    Test suite teardown
     Remove Drop In MicroShift Config    10-loglevel
+    Enable Journal Rate Limiting
     Restart MicroShift
     Logout MicroShift Host
     Remove Kubeconfig

test/suites/standard1/hostname.robot

@@ -13,7 +13,7 @@ Test Tags           restart    slow
 
 
 *** Variables ***
-${NEW_HOSTNAME}     microshift.local
+${NEW_HOSTNAME}     microshift-test.example
 ${OLD_HOSTNAME}     ${EMPTY}
 
 
@@ -29,7 +29,7 @@ Verify Local Host Name
     Should Contain    ${hostname}    standard
 
 Verify Local Host Name Resolution
-    [Documentation]    Verify correct host name resolution through mDNS
+    [Documentation]    Verify MicroShift restarts correctly after a hostname change
     [Setup]    Configure New Hostname
 
     Named Deployment Should Be Available    router-default    timeout=${DEFAULT_WAIT_TIMEOUT}    ns=openshift-ingress