From b9eae51473b48e429714df5f654085ddae305a4c Mon Sep 17 00:00:00 2001 From: Geethapranay1 Date: Fri, 27 Feb 2026 08:14:27 +0530 Subject: [PATCH 1/6] Add --suites filter to generate_reference_results Closes #711 --- .../generate_reference_results_manual.yml | 6 +++++ .../generate_reference_results_workflow.yml | 25 ++++++++++++++++--- changelog-entries/711.md | 1 + tools/tests/generate_reference_results.py | 23 ++++++++++++++++- 4 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 changelog-entries/711.md diff --git a/.github/workflows/generate_reference_results_manual.yml b/.github/workflows/generate_reference_results_manual.yml index f456b7e43..5dbcf3f28 100644 --- a/.github/workflows/generate_reference_results_manual.yml +++ b/.github/workflows/generate_reference_results_manual.yml @@ -10,6 +10,11 @@ on: description: 'Commit msg for commit that adds the reference results' default: "Adding reference results" type: string + suites: + description: 'Comma-separated test suites to generate reference results for (leave empty for all)' + default: '' + required: false + type: string log_level: description: 'Logging verbosity level used for the systemtests' default: 'INFO' @@ -28,4 +33,5 @@ jobs: with: from_ref: ${{ inputs.from_ref }} commit_msg: ${{ inputs.commit_msg }} + suites: ${{ inputs.suites }} log_level: ${{ inputs.log_level }} \ No newline at end of file diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml index 4b647ba97..4ef5b38ea 100644 --- a/.github/workflows/generate_reference_results_workflow.yml +++ b/.github/workflows/generate_reference_results_workflow.yml @@ -10,6 +10,11 @@ on: description: 'Commit msg for commit that adds the reference results' default: "Adding reference results" type: string + suites: + description: 'Comma-separated test suites to generate reference results for. If empty, all suites are generated.' + default: '' + required: false + type: string log_level: description: 'Logging verbosity level used for the systemtests' default: 'INFO' @@ -24,6 +29,7 @@ jobs: echo "Initiated by: ${{ github.actor }}" echo "Running generate_reference_results.py --log-level ${{inputs.log_level}}" echo "Using Ref: ${{ inputs.from_ref }}" + echo "Suites filter: ${{ inputs.suites || 'all (no filter)' }}" echo "Commit message on success: ${{ inputs.commit_msg }}" - name: Move LFS URL to local LFS server run: | @@ -56,15 +62,26 @@ jobs: test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py echo "Selected $GENERATE_REF_RESULTS to run" - python $GENERATE_REF_RESULTS --log-level=${{inputs.log_level}} + SUITES_ARG="" + if [ -n "${{ inputs.suites }}" ]; then + SUITES_ARG="--suites=${{ inputs.suites }}" + fi + python $GENERATE_REF_RESULTS --log-level=${{inputs.log_level}} $SUITES_ARG cd ../../ - name: Create commit if: success() run: | git checkout ${{ inputs.from_ref }} - git add ./*/*/*.tar.gz - git add ./*/*.tar.gz - git add ./*/reference-results/*.metadata + # Add only the changed reference results. + # When no suite filter is given, add all reference result files. + # When a suite filter is given, only add files that were actually modified. + if [ -z "${{ inputs.suites }}" ]; then + git add ./*/*/*.tar.gz + git add ./*/*.tar.gz + git add ./*/reference-results/*.metadata + else + git add --update '*.tar.gz' '*.metadata' + fi git commit -m "${{inputs.commit_msg}}" git push - name: Upload artifacts for debugging diff --git a/changelog-entries/711.md b/changelog-entries/711.md new file mode 100644 index 000000000..c0c7dacd9 --- /dev/null +++ b/changelog-entries/711.md @@ -0,0 +1 @@ +- Added optional `--suites` filter to `generate_reference_results.py` and the corresponding workflow, allowing selective regeneration of reference results for specific test suites instead of always regenerating all of them. https://github.com/precice/tutorials/pull/711 diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..4868279f9 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -86,6 +86,9 @@ def main(): parser = argparse.ArgumentParser(description='Generate reference data for systemtests') parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) + parser.add_argument('--suites', type=str, + help='Comma-separated test suites to generate reference results for. ' + 'If not specified, all suites are used.') parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Set the logging level') @@ -99,7 +102,25 @@ def main(): available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) - test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + all_test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + + if args.suites: + test_suites_requested = args.suites.split(',') + test_suites = [] + for name in test_suites_requested: + found = all_test_suites.get_by_name(name) + if not found: + logging.error(f"Did not find the testsuite with name {name}") + else: + test_suites.append(found) + if not test_suites: + raise RuntimeError( + f"No matching test suites with names {test_suites_requested} found. " + "Use print_test_suites.py to get an overview") + logging.info(f"Filtering to requested suites: {[s.name for s in test_suites]}") + else: + test_suites = all_test_suites + logging.info("No --suites filter specified, generating reference results for all suites.") # Read in parameters build_args = SystemtestArguments.from_yaml(PRECICE_TESTS_DIR / "reference_versions.yaml") From ff64154dbcdb87825fd632d91b285b02e792b2ba Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Tue, 12 May 2026 15:10:33 +0200 Subject: [PATCH 2/6] Improve parsing of test suites input Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tools/tests/generate_reference_results.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 564241e7f..aeac882cc 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -105,18 +105,31 @@ def main(): all_test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) if args.suites: - test_suites_requested = args.suites.split(',') + test_suites_requested = [] + for name in args.suites.split(','): + normalized_name = name.strip() + if normalized_name and normalized_name not in test_suites_requested: + test_suites_requested.append(normalized_name) + + if not test_suites_requested: + parser.error( + "The --suites option did not contain any valid suite names after parsing. " + "Use print_test_suites.py to get an overview") + test_suites = [] + unknown_test_suites = [] for name in test_suites_requested: found = all_test_suites.get_by_name(name) if not found: - logging.error(f"Did not find the testsuite with name {name}") + unknown_test_suites.append(name) else: test_suites.append(found) - if not test_suites: - raise RuntimeError( - f"No matching test suites with names {test_suites_requested} found. " + + if unknown_test_suites: + parser.error( + f"Unknown test suite name(s): {unknown_test_suites}. " "Use print_test_suites.py to get an overview") + logging.info(f"Filtering to requested suites: {[s.name for s in test_suites]}") else: test_suites = all_test_suites From 4ed3ef409967d7b6f6e4bfeb15bb3dedc2e009d3 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Tue, 12 May 2026 15:19:06 +0200 Subject: [PATCH 3/6] Improve error message in generate_reference_results.py --- tools/tests/generate_reference_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index aeac882cc..e3ac5261e 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -127,7 +127,7 @@ def main(): if unknown_test_suites: parser.error( - f"Unknown test suite name(s): {unknown_test_suites}. " + f"Unknown test suite name(s): {', '.join(unknown_test_suites)}. " "Use print_test_suites.py to get an overview") logging.info(f"Filtering to requested suites: {[s.name for s in test_suites]}") From 0f5c10b3e19a06446e72557995a5943617f077d5 Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Tue, 12 May 2026 15:20:36 +0200 Subject: [PATCH 4/6] Improve git add filter Suggested by Copilot --- .github/workflows/generate_reference_results_workflow.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml index 4ef5b38ea..e57966dbd 100644 --- a/.github/workflows/generate_reference_results_workflow.yml +++ b/.github/workflows/generate_reference_results_workflow.yml @@ -74,13 +74,14 @@ jobs: git checkout ${{ inputs.from_ref }} # Add only the changed reference results. # When no suite filter is given, add all reference result files. - # When a suite filter is given, only add files that were actually modified. + # When a suite filter is given, only add matching reference result artifacts, + # including newly created files for newly added tutorials or suites. if [ -z "${{ inputs.suites }}" ]; then git add ./*/*/*.tar.gz git add ./*/*.tar.gz git add ./*/reference-results/*.metadata else - git add --update '*.tar.gz' '*.metadata' + git add -A -- '*.tar.gz' '*.metadata' fi git commit -m "${{inputs.commit_msg}}" git push From 837037a21891c56275e1027fd9855fd60f6bcd4c Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Tue, 12 May 2026 15:21:48 +0200 Subject: [PATCH 5/6] Actions workflow: Improve input argument handling As suggested by Copilot --- .github/workflows/generate_reference_results_workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml index e57966dbd..21e594a69 100644 --- a/.github/workflows/generate_reference_results_workflow.yml +++ b/.github/workflows/generate_reference_results_workflow.yml @@ -62,11 +62,11 @@ jobs: test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py echo "Selected $GENERATE_REF_RESULTS to run" - SUITES_ARG="" + SUITES_ARGS=() if [ -n "${{ inputs.suites }}" ]; then - SUITES_ARG="--suites=${{ inputs.suites }}" + SUITES_ARGS+=(--suites "${{ inputs.suites }}") fi - python $GENERATE_REF_RESULTS --log-level=${{inputs.log_level}} $SUITES_ARG + python "$GENERATE_REF_RESULTS" --log-level="${{inputs.log_level}}" "${SUITES_ARGS[@]}" cd ../../ - name: Create commit if: success() From 2b81bd802e68103ce85c89af2dd7bb3d9a3b198a Mon Sep 17 00:00:00 2001 From: Gerasimos Chourdakis Date: Tue, 12 May 2026 15:29:15 +0200 Subject: [PATCH 6/6] Apply Copilot suggestion to improve git add Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/generate_reference_results_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml index 21e594a69..244b49e78 100644 --- a/.github/workflows/generate_reference_results_workflow.yml +++ b/.github/workflows/generate_reference_results_workflow.yml @@ -81,7 +81,7 @@ jobs: git add ./*/*.tar.gz git add ./*/reference-results/*.metadata else - git add -A -- '*.tar.gz' '*.metadata' + git add -A -- '**/reference-results/*.tar.gz' '**/reference-results/*.metadata' fi git commit -m "${{inputs.commit_msg}}" git push