From b237a966bbb37df867144738e964c146d52f2f0d Mon Sep 17 00:00:00 2001 From: Daniel Gerber <394442-gerbsen@users.noreply.gitlab.com> Date: Fri, 20 Feb 2026 14:57:27 +0100 Subject: [PATCH 1/2] Add option to run tests in parallel (as default) --- .gitignore | 2 ++ Tests/README.md | 15 ++++++++++++--- Tests/scs-compatible-kaas.yaml | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index c4f52eb2c..79fd578df 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ Tests/kaas/kaas-sonobuoy-tests/results/ *.tar.gz .direnv .vscode +Tests/sono-* +report.yaml diff --git a/Tests/README.md b/Tests/README.md index cfde06a74..946533944 100644 --- a/Tests/README.md +++ b/Tests/README.md @@ -49,10 +49,15 @@ Install KaaS-specific requirements: pip install -r kaas/requirements.txt ``` -Given a kubeconfig file `path/to/kubeconfig.yaml`, run +Given a kubeconfig file `path/to/kubeconfig.yaml`, run all tests in parallel with: ```shell -./scs-compliance-check.py -v -a kubeconfig=path/to/kubeconfig.yaml -a subject_root=. -s SUBJECT -o report.yaml scs-compatible-kaas.yaml +./scs-compliance-check.py -v -a kubeconfig=path/to/kubeconfig.yaml \ + -a subject_root=. \ + -a e2e-parallel="--e2e-parallel=true" \ + -s SUBJECT \ + -o report.yaml \ + scs-compatible-kaas.yaml ``` Replace `SUBJECT` with an arbitrary, but meaningful subject name. Also, please note that the check @@ -64,6 +69,10 @@ Additionally, the directory `sono-results` will be generated. It contains a JUni `plugins/e2e/results/global/junit_01.xml`. You can render it to HTML with a tool like junit2html. This might give you hints as to why a test failed. +If you need to run tests without parallelization simply remove `-a e2e-parallel="--e2e-parallel=true"` +or set it to `false`. +Some additional information can be found [here](https://github.com/vmware-tanzu/sonobuoy/issues/1435). + ## Usage information (help output) ```text @@ -183,7 +192,7 @@ We use a **layered approach** to allow for selective installation: - at the very top we have `test-requirements.in`. Whenever you change or recompile one of these layers, -*all layers above that layer have to be recompiled as well*. +_all layers above that layer have to be recompiled as well_. Note: The Python version used for running `pip-compile` should be consistent. The currently used version is documented in the header of the `requirements.txt`. It should match the diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 74acdb1a7..bfa3d243f 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -11,7 +11,7 @@ variables: - kubeconfig scripts: - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py - args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance' -a '--mode=certified-conformance' + args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance' -a '--mode=certified-conformance {e2e-parallel}' #~ args: -k {kubeconfig} -r {subject_root}/sono-results -c 'cncf-k8s-conformance' -a '--plugin-env e2e.E2E_DRYRUN=true' testcases: - id: cncf-k8s-conformance @@ -31,7 +31,7 @@ scripts: description: Must fulfill all requirements of scs-0214-v2. url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution#decision - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py - args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' + args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy" {e2e-parallel}' testcases: - id: kaas-networking-check description: Must fulfill all requirements of scs-0219-v1. From 6f8f7f5e5103a353b528fdcbbaf7a4fe634e8086 Mon Sep 17 00:00:00 2001 From: Daniel Gerber <394442-gerbsen@users.noreply.gitlab.com> Date: Mon, 23 Feb 2026 10:40:25 +0100 Subject: [PATCH 2/2] Make parallel execution the default but disableable --- Tests/README.md | 6 ++---- Tests/scs-compliance-check.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Tests/README.md b/Tests/README.md index 946533944..8f8de0f44 100644 --- a/Tests/README.md +++ b/Tests/README.md @@ -49,12 +49,11 @@ Install KaaS-specific requirements: pip install -r kaas/requirements.txt ``` -Given a kubeconfig file `path/to/kubeconfig.yaml`, run all tests in parallel with: +Given a kubeconfig file `path/to/kubeconfig.yaml`, run all tests (in parallel) with: ```shell ./scs-compliance-check.py -v -a kubeconfig=path/to/kubeconfig.yaml \ -a subject_root=. \ - -a e2e-parallel="--e2e-parallel=true" \ -s SUBJECT \ -o report.yaml \ scs-compatible-kaas.yaml @@ -69,8 +68,7 @@ Additionally, the directory `sono-results` will be generated. It contains a JUni `plugins/e2e/results/global/junit_01.xml`. You can render it to HTML with a tool like junit2html. This might give you hints as to why a test failed. -If you need to run tests without parallelization simply remove `-a e2e-parallel="--e2e-parallel=true"` -or set it to `false`. +If you need to run tests without parallelization simply add `-a e2e-parallel="--e2e-parallel=false"`. Some additional information can be found [here](https://github.com/vmware-tanzu/sonobuoy/issues/1435). ## Usage information (help output) diff --git a/Tests/scs-compliance-check.py b/Tests/scs-compliance-check.py index e23f04e29..a6d36f58f 100755 --- a/Tests/scs-compliance-check.py +++ b/Tests/scs-compliance-check.py @@ -90,7 +90,7 @@ def __init__(self): self.verbose = False self.quiet = False self.subject = "" - self.assignment = {} + self.assignment = { "e2e-parallel" : "--e2e-parallel=true" } self.checkdate = datetime.date.today() self.version = None self.output = None @@ -135,7 +135,7 @@ def apply_argv(self, argv): self.critical_only = True elif opt[0] == "-a" or opt[0] == "--assign": key, value = opt[1].split("=", 1) - if key in self.assignment: + if key in self.assignment and key != "e2e-parallel": raise ValueError(f"Double assignment for {key!r}") self.assignment[key] = value elif opt[0] == "-t" or opt[0] == "--tests":