Ben/functional splits

.cron.yml

@@ -39,3 +39,34 @@ jobs:
     when:
       - { hour: 11, minute: 15 }
       - { hour: 23, minute: 15 }
+
+  - name: new-beta-func
+    job:
+      type: decision-task
+      treeherder-symbol: STARb-func
+      target-tasks-method: new_beta_func
+    when:
+      - { hour: 0, minute: 45 }
+      - { hour: 1, minute: 45 }
+      - { hour: 2, minute: 45 }
+      - { hour: 3, minute: 45 }
+      - { hour: 4, minute: 45 }
+      - { hour: 5, minute: 45 }
+      - { hour: 6, minute: 45 }
+      - { hour: 7, minute: 45 }
+      - { hour: 8, minute: 45 }
+      - { hour: 9, minute: 45 }
+      - { hour: 10, minute: 45 }
+      - { hour: 11, minute: 45 }
+      - { hour: 12, minute: 45 }
+      - { hour: 13, minute: 45 }
+      - { hour: 14, minute: 45 }
+      - { hour: 15, minute: 45 }
+      - { hour: 16, minute: 45 }
+      - { hour: 17, minute: 45 }
+      - { hour: 18, minute: 45 }
+      - { hour: 19, minute: 45 }
+      - { hour: 20, minute: 45 }
+      - { hour: 21, minute: 45 }
+      - { hour: 22, minute: 45 }
+      - { hour: 23, minute: 45 }

.github/workflows/check-beta-functional.yml

@@ -2,14 +2,23 @@ name: Check new beta version - Functional
 
 on:
   schedule:
-    - cron: "55 1/5 1 * *"
+    - cron: "55 * * * *"
 env:
   LATEST: ""
 permissions:
   contents: "write"
 
 jobs:
+  Get-Functional-Split:
+    runs-on: ubuntu-latest
+    outputs: ${{ steps.getsplit.outputs.split }}
+    steps:
+      - name: Get Split Based On Time
+        id: getsplit
+        run: echo "split=functional$(date +%H)" >> "$GITHUB_OUTPUT"
+
   Check-Beta-Version:
+    needs: Get-Functional-Split
     runs-on: ubuntu-latest
     outputs:
       win_reportable: ${{ steps.reportable.outputs.win }}
@@ -23,7 +32,7 @@ jobs:
       - name: Check if the run is reportable
         id: reportable
         env:
-          STARFOX_SPLIT: functional1
+          STARFOX_SPLIT: ${{ steps.getsplit.outputs.split }}
           TESTRAIL_REPORT: true
           TESTRAIL_BASE_URL: ${{ secrets.TESTRAIL_BASE_URL }}
           TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
@@ -40,7 +49,7 @@ jobs:
     uses: ./.github/workflows/main.yml
     with:
       job_to_run: Test-Windows
-      test_set: functional
+      test_set: ${{ steps.getsplit.outputs.split }}
     secrets: inherit
 
   Run-Mac-Functional:
@@ -49,5 +58,5 @@ jobs:
     uses: ./.github/workflows/main.yml
     with:
       job_to_run: Test-MacOS
-      test_set: functional
+      test_set: ${{ steps.getsplit.outputs.split }}
     secrets: inherit

.github/workflows/main.yml

@@ -70,10 +70,6 @@ jobs:
         run: |
           echo "STARFOX_SPLIT=$TEST_SET" | Out-File -FilePath .env -Encoding utf8 -Append -Force
         shell: pwsh
-      - name: Set Functional Split
-        if: ${{ inputs.test_set == 'functional' }}
-        run: |
-          echo "STARFOX_SPLIT=$TEST_SET1" | Out-File -FilePath .env -Encoding utf8 -Append -Force
       - name: Install dependencies
         run: |
           mkdir -p artifacts;
@@ -175,9 +171,6 @@ jobs:
       - name: Select Test Set
         if: ${{ inputs.test_set != null }}
         run: echo "STARFOX_SPLIT=$TEST_SET" >> .env
-      - name: Set Functional Split
-        if: ${{ inputs.test_set == 'functional' }}
-        run: echo "STARFOX_SPLIT=$TEST_SET1" >> .env
       - name: Set Environment (Manual)
         if: ${{ inputs.mac_installer_link }}
         run: |

manifests/testkey.py

@@ -6,7 +6,7 @@
 
 import yaml
 
-NUM_FUNCTIONAL_SPLITS = 1
+NUM_FUNCTIONAL_SPLITS = 2
 MAX_DEPTH = 5
 SUITE_TUPLE_RE = re.compile(r'\s+return \("S(\d+)", ?".*"\)')
 
@@ -125,7 +125,9 @@ def get_entry_from_filename(self, filename) -> dict:
     def rebalance_functionals(self):
         all_functs = []
         for splitnum in range(NUM_FUNCTIONAL_SPLITS):
-            all_functs.extend(self.gather_split(f"functional{splitnum}"))
+            all_functs.extend(
+                self.gather_split(f"functional{splitnum}", pass_only=False)
+            )
         total_functs = len(all_functs)
         functs_per_split = total_functs // NUM_FUNCTIONAL_SPLITS
         remainder = total_functs % NUM_FUNCTIONAL_SPLITS
@@ -139,19 +141,25 @@ def rebalance_functionals(self):
             splitnum = 0
             for j in range(NUM_FUNCTIONAL_SPLITS):
                 if i > split_cutoffs[j]:
-                    splitnum = j
+                    splitnum = j + 1
             entry = self.get_entry_from_filename(test_filename)
-            ptr = entry
+            suite = list(entry.keys())[0]
+            ptr = entry[suite]
             for _ in range(MAX_DEPTH):
                 if "splits" not in ptr:
-                    ptr = entry[list(entry.keys())[0]]
+                    ptr = ptr[list(ptr.keys())[0]]
+                else:
+                    break
+            actual_test = list(ptr.keys())[0]
             if f"functional{splitnum + 1}" in ptr["splits"]:
+                print(f"Keeping {actual_test} in functional split {splitnum + 1}")
                 continue
             for split_ in ptr["splits"]:
                 if split_.startswith("functional"):
                     ptr["splits"].remove(split_)
+            print(f"Moving {actual_test} to functional split {splitnum + 1}")
             ptr["splits"].append(f"functional{splitnum + 1}")
-            self.manifest |= entry
+            self.manifest[suite] |= entry[suite]
 
     def gather_split(self, split_name, pass_only=True):
         # Given a split name, return the pytest locations of the tests in that split

modules/testrail_integration.py

@@ -297,6 +297,9 @@ def reportable(platform_to_test=None):
         expected_suites = manifest.get_valid_suites_in_split(
             os.environ["STARFOX_SPLIT"], suite_numbers=True
         )
+        if not expected_suites:
+            logging.warning("This split is empty, not running or reporting.")
+            return False
 
         uncovered_suites = list(set(expected_suites) - set(covered_suites))
         if len(uncovered_suites):

taskcluster/kinds/new-beta-func/kind.yml

@@ -32,7 +32,7 @@ tasks:
       using: run-task
       cwd: "{checkout}"
       command: |-
-        export STARFOX_SPLIT="${STARFOX_SPLIT}1"
+        export STARFOX_SPLIT="${STARFOX_SPLIT}$(date +%H)"
         export PATH=$HOME/.local/bin:$PWD:$PATH
         mkdir -p artifacts;
         pip3 install 'pipenv==2023.11.15';