From 58bb510a9295a13faf4739a10411c9dba745180a Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 12 Dec 2025 11:03:36 -0800 Subject: [PATCH 1/4] fix rebalance code --- manifests/testkey.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/manifests/testkey.py b/manifests/testkey.py index 1581611a8..54a1ef3f9 100644 --- a/manifests/testkey.py +++ b/manifests/testkey.py @@ -6,7 +6,7 @@ import yaml -NUM_FUNCTIONAL_SPLITS = 1 +NUM_FUNCTIONAL_SPLITS = 2 MAX_DEPTH = 5 SUITE_TUPLE_RE = re.compile(r'\s+return \("S(\d+)", ?".*"\)') @@ -125,7 +125,9 @@ def get_entry_from_filename(self, filename) -> dict: def rebalance_functionals(self): all_functs = [] for splitnum in range(NUM_FUNCTIONAL_SPLITS): - all_functs.extend(self.gather_split(f"functional{splitnum}")) + all_functs.extend( + self.gather_split(f"functional{splitnum}", pass_only=False) + ) total_functs = len(all_functs) functs_per_split = total_functs // NUM_FUNCTIONAL_SPLITS remainder = total_functs % NUM_FUNCTIONAL_SPLITS @@ -139,19 +141,25 @@ def rebalance_functionals(self): splitnum = 0 for j in range(NUM_FUNCTIONAL_SPLITS): if i > split_cutoffs[j]: - splitnum = j + splitnum = j + 1 entry = self.get_entry_from_filename(test_filename) - ptr = entry + suite = list(entry.keys())[0] + ptr = entry[suite] for _ in range(MAX_DEPTH): if "splits" not in ptr: - ptr = entry[list(entry.keys())[0]] + ptr = ptr[list(ptr.keys())[0]] + else: + break + actual_test = list(ptr.keys())[0] if f"functional{splitnum + 1}" in ptr["splits"]: + print(f"Keeping {actual_test} in functional split {splitnum + 1}") continue for split_ in ptr["splits"]: if split_.startswith("functional"): ptr["splits"].remove(split_) + print(f"Moving {actual_test} to functional split {splitnum + 1}") ptr["splits"].append(f"functional{splitnum + 1}") - self.manifest |= entry + self.manifest[suite] |= entry[suite] def gather_split(self, split_name, pass_only=True): # Given a split name, return the pytest locations of the tests in that split From 26a67964dd470cfa3f17575dbb1efd6b95572ebf Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 12 Dec 2025 11:17:51 -0800 Subject: [PATCH 2/4] set gha to run multiple functional splits --- .github/workflows/check-beta-functional.yml | 17 +++++++++++++---- .github/workflows/main.yml | 7 ------- modules/testrail_integration.py | 2 ++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/check-beta-functional.yml b/.github/workflows/check-beta-functional.yml index 3b1f121c9..5ffe86d4b 100644 --- a/.github/workflows/check-beta-functional.yml +++ b/.github/workflows/check-beta-functional.yml @@ -2,14 +2,23 @@ name: Check new beta version - Functional on: schedule: - - cron: "55 1/5 1 * *" + - cron: "55 * * * *" env: LATEST: "" permissions: contents: "write" jobs: + Get-Functional-Split: + runs-on: ubuntu-latest + outputs: ${{ steps.getsplit.outputs.split }} + steps: + - name: Get Split Based On Time + id: getsplit + run: echo "split=functional$(date +%H)" >> "$GITHUB_OUTPUT" + Check-Beta-Version: + needs: Get-Functional-Split runs-on: ubuntu-latest outputs: win_reportable: ${{ steps.reportable.outputs.win }} @@ -23,7 +32,7 @@ jobs: - name: Check if the run is reportable id: reportable env: - STARFOX_SPLIT: functional1 + STARFOX_SPLIT: ${{ steps.getsplit.outputs.split }} TESTRAIL_REPORT: true TESTRAIL_BASE_URL: ${{ secrets.TESTRAIL_BASE_URL }} TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} @@ -40,7 +49,7 @@ jobs: uses: ./.github/workflows/main.yml with: job_to_run: Test-Windows - test_set: functional + test_set: ${{ steps.getsplit.outputs.split }} secrets: inherit Run-Mac-Functional: @@ -49,5 +58,5 @@ jobs: uses: ./.github/workflows/main.yml with: job_to_run: Test-MacOS - test_set: functional + test_set: ${{ steps.getsplit.outputs.split }} secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d773a91a..8ab4fdd9a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,10 +70,6 @@ jobs: run: | echo "STARFOX_SPLIT=$TEST_SET" | Out-File -FilePath .env -Encoding utf8 -Append -Force shell: pwsh - - name: Set Functional Split - if: ${{ inputs.test_set == 'functional' }} - run: | - echo "STARFOX_SPLIT=$TEST_SET1" | Out-File -FilePath .env -Encoding utf8 -Append -Force - name: Install dependencies run: | mkdir -p artifacts; @@ -175,9 +171,6 @@ jobs: - name: Select Test Set if: ${{ inputs.test_set != null }} run: echo "STARFOX_SPLIT=$TEST_SET" >> .env - - name: Set Functional Split - if: ${{ inputs.test_set == 'functional' }} - run: echo "STARFOX_SPLIT=$TEST_SET1" >> .env - name: Set Environment (Manual) if: ${{ inputs.mac_installer_link }} run: | diff --git a/modules/testrail_integration.py b/modules/testrail_integration.py index 6390c5014..25c786134 100644 --- a/modules/testrail_integration.py +++ b/modules/testrail_integration.py @@ -297,6 +297,8 @@ def reportable(platform_to_test=None): expected_suites = manifest.get_valid_suites_in_split( os.environ["STARFOX_SPLIT"], suite_numbers=True ) + if not expected_suites: + logging.warning("This split is empty, not running or reporting.") uncovered_suites = list(set(expected_suites) - set(covered_suites)) if len(uncovered_suites): From 276cfe8034ff1b14e84efeb130d9434fbc2d121e Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 12 Dec 2025 11:32:22 -0800 Subject: [PATCH 3/4] set tc to run multiple functional splits --- .cron.yml | 31 ++++++++++++++++++++++++ taskcluster/kinds/new-beta-func/kind.yml | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.cron.yml b/.cron.yml index 5011bbcbf..5856b6506 100644 --- a/.cron.yml +++ b/.cron.yml @@ -39,3 +39,34 @@ jobs: when: - { hour: 11, minute: 15 } - { hour: 23, minute: 15 } + + - name: new-beta-func + job: + type: decision-task + treeherder-symbol: STARb-func + target-tasks-method: new_beta_func + when: + - { hour: 0, minute: 45 } + - { hour: 1, minute: 45 } + - { hour: 2, minute: 45 } + - { hour: 3, minute: 45 } + - { hour: 4, minute: 45 } + - { hour: 5, minute: 45 } + - { hour: 6, minute: 45 } + - { hour: 7, minute: 45 } + - { hour: 8, minute: 45 } + - { hour: 9, minute: 45 } + - { hour: 10, minute: 45 } + - { hour: 11, minute: 45 } + - { hour: 12, minute: 45 } + - { hour: 13, minute: 45 } + - { hour: 14, minute: 45 } + - { hour: 15, minute: 45 } + - { hour: 16, minute: 45 } + - { hour: 17, minute: 45 } + - { hour: 18, minute: 45 } + - { hour: 19, minute: 45 } + - { hour: 20, minute: 45 } + - { hour: 21, minute: 45 } + - { hour: 22, minute: 45 } + - { hour: 23, minute: 45 } diff --git a/taskcluster/kinds/new-beta-func/kind.yml b/taskcluster/kinds/new-beta-func/kind.yml index 4c1417920..f63a67b8f 100644 --- a/taskcluster/kinds/new-beta-func/kind.yml +++ b/taskcluster/kinds/new-beta-func/kind.yml @@ -32,7 +32,7 @@ tasks: using: run-task cwd: "{checkout}" command: |- - export STARFOX_SPLIT="${STARFOX_SPLIT}1" + export STARFOX_SPLIT="${STARFOX_SPLIT}$(date +%H)" export PATH=$HOME/.local/bin:$PWD:$PATH mkdir -p artifacts; pip3 install 'pipenv==2023.11.15'; From 630cbfbe745c8e143f8f93f7b7e01503110df261 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 12 Dec 2025 13:15:04 -0800 Subject: [PATCH 4/4] bail on split not found --- modules/testrail_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/testrail_integration.py b/modules/testrail_integration.py index 25c786134..04d62bc8e 100644 --- a/modules/testrail_integration.py +++ b/modules/testrail_integration.py @@ -299,6 +299,7 @@ def reportable(platform_to_test=None): ) if not expected_suites: logging.warning("This split is empty, not running or reporting.") + return False uncovered_suites = list(set(expected_suites) - set(covered_suites)) if len(uncovered_suites):