From 09cc2fd9fdba3d832e8da5ebe310d7ef26b507b4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:00:14 -0400 Subject: [PATCH 1/4] fix(ci-status): leave no red yielded sibling after a full run succeeds A yielding contract-only run could stay red with nothing left to re-run it in two ways: - It read the full run's success while that run's aggregate step was still running, after its last sibling scan, and failed superseded. It now re-lists and re-reads every 5s for up to 30s until that step finishes, then carries the success once the step reads back success. A running step still never proves the status, so nothing passes sooner. - It counted a contract-only sibling's re-run attempt, whose new jobs were not listed yet, as a full run. A re-run keeps its event, so such an attempt is now classified by its first attempt's jobs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../aggregate-results/action.yml | 18 ++- .../aggregate-results/run.sh | 128 +++++++++++++----- .../aggregate-results/run.test.sh | 119 ++++++++++++++++ README.md | 28 +++- 4 files changed, 250 insertions(+), 43 deletions(-) diff --git a/.github/actions/pr-require-checks/aggregate-results/action.yml b/.github/actions/pr-require-checks/aggregate-results/action.yml index 5cfc0cf..69aad3f 100644 --- a/.github/actions/pr-require-checks/aggregate-results/action.yml +++ b/.github/actions/pr-require-checks/aggregate-results/action.yml @@ -158,19 +158,27 @@ inputs: default: 'false' yield-to-full-run: description: >- - `true` to have a contract-only run never wait: it lists this workflow's - in-flight runs on the head SHA once, then reads the `status-context` - status once. A full run in flight supersedes it, so it fails at once + `true` to have a contract-only run not wait for a full run's verdict: it + lists this workflow's in-flight runs on the head SHA, then reads the + `status-context` status. A full run in flight supersedes it, so it fails at once with `superseded by full run `: that run's own `ci-status` check run is newer and decides the merge gate, and with `rerun-contract-only-siblings` it re-runs this run once it records `success`. With no full run in flight it passes only on a recorded `success`. An in-flight sibling whose latest attempt skipped every job but one, and that one has this run's gate name, is contract-only and - ignored; any other sibling, including one whose jobs cannot be read yet, + ignored, and so is a re-run attempt whose first attempt had that shape + (a re-run keeps its event); any other sibling, including one whose jobs + cannot be read yet, counts as a full run, so a misread goes red, never green. The writer of a `success` already on the SHA is ignored while the attempt that wrote - it finishes. Overrides + it finishes. When the newest status is a `success` written by the + current attempt of the one full run in flight, and that run's step that + wrote it is still running (re-running failed siblings), it re-lists and + re-reads every 5 seconds for up to 30 until that step finishes, then + carries the `success` once the step reads back `success`; a step still + running at 30 seconds fails superseded, and a full run still waiting on + this run re-runs it. Overrides `carry-forward-wait-seconds`. Closes the gap `record-pending` alone leaves, a full run queued before its `pending` marker is written. The calling job needs `actions: read`; a listing that fails after three diff --git a/.github/actions/pr-require-checks/aggregate-results/run.sh b/.github/actions/pr-require-checks/aggregate-results/run.sh index d0678a0..0368caf 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.sh @@ -124,6 +124,12 @@ CARRY_FORWARD_POLL_SECONDS=15 # above. RERUN_WAIT_SECONDS="${RERUN_WAIT_SECONDS:-90}" RERUN_WAIT_POLL_SECONDS=10 +# Ceiling, in seconds, on a yielding run's wait for the step of the full run +# that recorded `success` to finish; see the yield block. Below the default +# `rerun-wait-seconds` on purpose: when that full run is waiting on this run, +# this run gives up first, and the full run re-runs it. Fixed, like the polls. +YIELD_WRITER_WAIT_SECONDS=30 +YIELD_WRITER_POLL_SECONDS=5 # GitHub run statuses that mean "this run has not finished yet". `completed` is # the only other value, and a completed run either wrote the status or never # will. @@ -315,6 +321,9 @@ carried_state="" # treats a run as the writer's re-run. carried_created_at="" carried_writer_run_id="" +# True when `carried_state` is `pending` only because the step of this workflow's +# run on this SHA that the `success` names is still running. +carried_writer_running=false # Whether `carried_state` holds a completed read. The poll loop reads the status # itself, so the caller below must not read a second time and overwrite what the # loop decided on. @@ -324,6 +333,7 @@ read_carried_state() { carried_state="" carried_created_at="" carried_writer_run_id="" + carried_writer_running=false carried_state_read=false # The LIST endpoint, not the combined one: `commits//status` collapses to # one entry per context and exposes no author, so any collaborator with write @@ -463,6 +473,7 @@ verify_writer_step() { running) echo "Run ${carried_writer_run_id} has not finished step '${own_gate_step}' of job '${own_gate_job}'; its ${STATUS_CONTEXT} success is not carried yet." carried_state=pending + carried_writer_running=true ;; *) echo "::warning::the newest ${STATUS_CONTEXT} success on ${SHA} names run ${carried_writer_run_id}, whose step '${own_gate_step}' of job '${own_gate_job}' did not succeed (${outcome:-no conclusion}); ignoring it." @@ -673,14 +684,26 @@ fail_carry_forward() { exit 1 } -# Yield mode (`yield-to-full-run` true): a contract-only run never waits. It -# lists this workflow's in-flight runs on the SHA once, then reads the status -# once, in that order for the reason the wait gives. A full run in flight -# supersedes this run: it fails at once naming that run, whose own ci-status -# check run is newer than this one's and is the one the merge gate reads. With -# `rerun-contract-only-siblings`, that run re-runs this one once it records -# `success`. A red is never a false green, so failing here is always safe; the -# cost is a red check run that the full run's newer one supersedes. +# Yield mode (`yield-to-full-run` true): a contract-only run does not wait for +# a full run's verdict. It lists this workflow's in-flight runs on the SHA, +# then reads the status, in that order for the reason the wait gives. A full +# run in flight supersedes this run: it fails at once naming that run, whose +# own ci-status check run is newer than this one's and is the one the merge +# gate reads. With `rerun-contract-only-siblings`, that run re-runs this one +# once it records `success`. A red is never a false green, so failing here is +# always safe; the cost is a red check run that the full run's newer one +# supersedes. +# +# One state waits, briefly: the newest status is a `success` naming the one +# full run in flight, written by its current attempt, whose aggregate step is +# still running. That step has already recorded the verdict and is only +# re-running failed siblings, after its last listing of them, so a red here +# would never be re-run. The success cannot be carried yet, because a running +# step cannot prove it wrote the status (see `verify_writer_step`), so this run +# re-lists and re-reads every YIELD_WRITER_POLL_SECONDS until that step +# finishes, up to YIELD_WRITER_WAIT_SECONDS, then decides as above. At the +# ceiling it fails superseded: a full run still waiting on this run re-runs it +# once it completes. # # This closes the gap `record-pending` alone leaves: a full run that is queued, # or whose first job has not yet written its `pending` marker, would otherwise @@ -694,7 +717,11 @@ fail_carry_forward() { # whose lanes were skipped, before its gate job exists, from passing for one. # The gate name is this run's own one non-skipped job; when that cannot be # read, every sibling is a full run, as is one with more jobs than the one page -# of 100 lists. Every other sibling is a full run, +# of 100 lists. A re-run attempt (`run_attempt` above 1) not in that shape yet +# is classified by its first attempt's jobs, since a re-run keeps its event: a +# full run re-runs failed contract-only runs, and counting one whose new jobs +# are not listed yet as a full run would fail this run with nothing left to +# re-run it. Every other sibling is a full run, # including one whose jobs are not listed yet or cannot be read, so a misread # fails closed. The one full run # excluded is the writer of a `success` already on the SHA, when the attempt in @@ -746,6 +773,18 @@ list_in_flight_full_runs() { else warn_actions_failed "repos/${REPOSITORY}/actions/runs/${id}/jobs" 'actions: read' "Counting run ${id} as a full run in flight." fi + # A re-run keeps its original event, so a re-run of a contract-only run is + # contract-only too, yet its new attempt's jobs may not be listed yet. Its + # first attempt has completed, so that attempt's jobs decide. + if [[ "$shape" != true ]] && + [[ "$(jq -r --arg id "$id" '[ .workflow_runs[]? | select((.id | tostring) == $id and (.run_attempt // 1) > 1) ] | length > 0' <"$scratch/runs.json")" == true ]]; then + # shellcheck disable=SC2310 # gh_api handles its own errexit; an unreadable first attempt leaves the run full. + if gh_api GET "repos/${REPOSITORY}/actions/runs/${id}/attempts/1/jobs?per_page=100"; then + shape="$(jq -r --arg gate "$gate" '(.total_count // 0) as $total | [ .jobs[]? ] | length > 1 and length >= $total and (map(select(.conclusion != "skipped") | .name) == [$gate])' <"$gh_stdout")" || shape=false + else + warn_actions_failed "repos/${REPOSITORY}/actions/runs/${id}/attempts/1/jobs" 'actions: read' "Counting run ${id} as a full run in flight." + fi + fi if [[ "$shape" != true ]]; then yield_full_runs="${yield_full_runs}${yield_full_runs:+ }${id}" fi @@ -768,31 +807,58 @@ fail_superseded() { exit 1 } +# True when run `$1` is the writer the carried status names and its attempt in +# flight started at or before that status was created, so that attempt wrote +# it. A listing jq cannot parse reads as false, which keeps the run superseding. +writer_attempt_wrote() { + [[ "$1" == "$carried_writer_run_id" && -n "$carried_created_at" ]] && + [[ "$(jq -r --arg id "$1" --arg since "$carried_created_at" \ + '[ .workflow_runs[]? | select((.id | tostring) == $id and (.run_started_at // "") != "" and .run_started_at <= $since) ] | length > 0' \ + <"$scratch/runs.json")" == true ]] +} + if [[ "$contract_only" == true && "$yield_to_full_run" == true ]]; then - echo "Contract-only event: yielding to any full run in flight on ${SHA}, else reading the ${STATUS_CONTEXT} status once." - # shellcheck disable=SC2310 # list_in_flight_full_runs warns itself; the caller fails closed. - if ! list_in_flight_full_runs; then - echo "::error::could not list this workflow's runs on ${SHA}, so a full run in flight cannot be ruled out; re-run this run." - exit 1 - fi - # shellcheck disable=SC2310 # read_carried_state handles its own errexit; the caller classifies the status. - if ! read_carried_state; then - cat "$gh_stderr" >&2 - fail_carry_forward - fi - superseding="" - for id in $yield_full_runs; do - if [[ "$carried_state" == success && "$id" == "$carried_writer_run_id" && -n "$carried_created_at" ]] && - [[ "$(jq -r --arg id "$id" --arg since "$carried_created_at" \ - '[ .workflow_runs[]? | select((.id | tostring) == $id and (.run_started_at // "") != "" and .run_started_at <= $since) ] | length > 0' \ - <"$scratch/runs.json")" == true ]]; then - continue + echo "Contract-only event: yielding to any full run in flight on ${SHA}, else reading the ${STATUS_CONTEXT} status." + yield_started="$(date +%s)" + while :; do + # shellcheck disable=SC2310 # list_in_flight_full_runs warns itself; the caller fails closed. + if ! list_in_flight_full_runs; then + echo "::error::could not list this workflow's runs on ${SHA}, so a full run in flight cannot be ruled out; re-run this run." + exit 1 + fi + # shellcheck disable=SC2310 # read_carried_state handles its own errexit; the caller classifies the status. + if ! read_carried_state; then + cat "$gh_stderr" >&2 + fail_carry_forward + fi + superseding="" + for id in $yield_full_runs; do + # shellcheck disable=SC2310 # writer_attempt_wrote reports through its status; false keeps the run superseding. + if [[ "$carried_state" == success ]] && writer_attempt_wrote "$id"; then + continue + fi + superseding="${superseding}${superseding:+ }${id}" + done + if [[ -z "$superseding" ]]; then + break + fi + # shellcheck disable=SC2310 # writer_attempt_wrote reports through its status; false fails superseded. + if [[ "$carried_writer_running" == true && "$superseding" == "$carried_writer_run_id" ]] && writer_attempt_wrote "$superseding"; then + yield_waited=$(($(date +%s) - yield_started)) + yield_remaining=$((YIELD_WRITER_WAIT_SECONDS - yield_waited)) + if [[ "$yield_remaining" -gt 0 ]]; then + yield_sleep="$YIELD_WRITER_POLL_SECONDS" + if [[ "$yield_sleep" -gt "$yield_remaining" ]]; then + yield_sleep="$yield_remaining" + fi + echo "Run ${superseding} recorded ${STATUS_CONTEXT}=success on ${SHA} and its step is still finishing; waiting ${yield_sleep}s (waited ${yield_waited}s of ${YIELD_WRITER_WAIT_SECONDS}s)." + sleep "$yield_sleep" + continue + fi + echo "::warning::the step of run ${superseding} that recorded ${STATUS_CONTEXT}=success on ${SHA} was still running after ${yield_waited}s." fi - superseding="${superseding}${superseding:+ }${id}" - done - if [[ -n "$superseding" ]]; then fail_superseded "$superseding" - fi + done if [[ "$carried_state" == success ]]; then echo "Carried forward: ${STATUS_CONTEXT} is success on ${SHA} (recorded by ${STATUS_CREATOR})." exit 0 diff --git a/.github/actions/pr-require-checks/aggregate-results/run.test.sh b/.github/actions/pr-require-checks/aggregate-results/run.test.sh index 9eb9319..11c40a9 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.test.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.test.sh @@ -1629,6 +1629,125 @@ jobs_raw 4000 "$full_run_jobs" run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true expect_log "${superseded_prefix}${superseded_automatic}" +# The late sibling: run 4000 recorded success and is still in its aggregate +# step, past its last listing of siblings, so nothing would re-run a red here. +# Without the wait this fails superseded by 4000; without re-reading after it, +# the step's completion is never seen. +writer_jobs_key='GET_repos_melodic-software_ci-workflows_actions_runs_4000_jobs' +writer_running_then_success() { + writer_jobs 4000 running + mv -- "$fixtures/${writer_jobs_key}.json" "$fixtures/${writer_jobs_key}.1.json" + cp -- "$fixtures/${writer_jobs_key}.1.json" "$fixtures/${writer_jobs_key}.2.json" + writer_jobs 4000 success +} +late_writer_runs="[$(attempt_entry 4000 in_progress 2026-09-05T12:00:00Z 1 2026-09-05T12:00:00Z)]" +echo 'case: yield carries a success whose writer step is still finishing once that step succeeds' +reset_yield_fixtures +status_list "[$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +workflow_runs "$late_writer_runs" +writer_running_then_success +run_case 0 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "Run 4000 recorded ci-lanes=success on ${sha} and its step is still finishing; waiting 5s (waited 0s of 30s)." +expect_log "Carried forward: ci-lanes is success on ${sha}" +expect_no_log 'superseded' +expect_status_reads 2 + +# Without the ceiling a writer step that never finishes (a cancelled run) would +# hold this run to the job timeout; without failing at it, the unproven +# success would pass. +echo 'case: yield fails superseded when the writer step is still running at the ceiling' +reset_yield_fixtures +status_list "[$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +workflow_runs "$late_writer_runs" +writer_jobs 4000 running +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "::warning::the step of run 4000 that recorded ci-lanes=success on ${sha} was still running after 30s." +expect_log "${superseded_prefix}${superseded_automatic}" +expect_no_log 'Carried forward' +if [[ "$(awk '{ total += $1 } END { print total + 0 }' "$sleep_log")" -ne 30 ]]; then + echo "FAIL: expected the writer wait to sleep 30s in total, got: $(paste -sd' ' "$sleep_log")" + failures=$((failures + 1)) +fi + +# Without keying the wait on the writer's running step, a `pending` marker newer +# than the success would be waited on as well; it means the lanes are running, +# and the full run re-runs this run once they pass. +echo 'case: yield fails at once on a pending marker newer than the success' +reset_yield_fixtures +status_list "[$(writer_status 101 pending 2026-09-05T12:06:00Z 4000),$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +workflow_runs "$late_writer_runs" +writer_jobs 4000 running +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "${superseded_prefix}${superseded_automatic}" +expect_no_sleep +expect_status_reads 1 + +# Without failing closed on the re-read, a status read that fails during the +# writer wait would leave the first read's state in place. +echo 'case: yield fails closed when the status re-read fails during the writer wait' +reset_yield_fixtures +status_list "[$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +workflow_runs "$late_writer_runs" +writer_running_then_success +printf '2\n' >"$fixtures/${statuses_key}.fail-on-call" +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "${fail_prefix}${absent_remedy}" +expect_no_log 'Carried forward' +rm -f -- "$fixtures/${statuses_key}.fail-on-call" + +echo 'case: yield fails closed when the status re-read cannot be parsed during the writer wait' +reset_yield_fixtures +status_list "[$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +status_list_on_call 2 'not json' +workflow_runs "$late_writer_runs" +writer_running_then_success +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "${fail_prefix}${absent_remedy}" +expect_no_log 'Carried forward' + +# provisioning run 37254877117: full run 4000 recorded success and, still in +# its aggregate step, re-ran the failed contract-only run 4300, whose attempt 2 +# lists only its queued gate so far. Without classifying a re-run attempt by +# its first attempt, 4300 counts as a full run and this run fails superseded by +# a run that never re-runs anything. +attempt_one_jobs() { + printf '{"total_count":%s,"jobs":%s}' "$(jq length <<<"$2")" "$2" \ + >"$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_${1}_attempts_1_jobs.json" +} +rerun_sibling_jobs='[{"name":"ci-status","conclusion":null}]' +echo 'case: yield ignores a re-run attempt of a contract-only sibling whose new jobs are not listed yet' +reset_yield_fixtures +status_list "[$(writer_status 100 success 2026-09-05T12:05:00Z 4000)]" +workflow_runs "[$(attempt_entry 4000 in_progress 2026-09-05T12:00:00Z 1 2026-09-05T12:00:00Z),$(attempt_entry 4300 queued 2026-09-05T12:00:31Z 2 2026-09-05T12:05:02Z)]" +writer_running_then_success +jobs_raw 4300 "$rerun_sibling_jobs" +attempt_one_jobs 4300 '[{"name":"ci-status","conclusion":"failure"},{"name":"lint","conclusion":"skipped"},{"name":"test","conclusion":"skipped"}]' +run_case 0 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRACT_ONLY_SIBLINGS=true +expect_log "Carried forward: ci-lanes is success on ${sha}" +expect_no_log 'superseded' + +# Without reading the first attempt's shape, rather than the attempt number +# alone, a re-run of a full run would be ignored and an older success carried +# across it. +echo 'case: yield is superseded by a re-run attempt of a full run' +reset_yield_fixtures +status_list "[$(full_run_success 100)]" +workflow_runs "[$(attempt_entry 4300 queued 2026-09-05T12:00:31Z 2 2026-09-05T12:05:02Z)]" +jobs_raw 4300 "$rerun_sibling_jobs" +attempt_one_jobs 4300 '[{"name":"changes","conclusion":"success"},{"name":"lint","conclusion":"failure"},{"name":"ci-status","conclusion":"failure"}]' +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true +expect_log '::error::superseded by full run https://github.com/melodic-software/ci-workflows/actions/runs/4300' + +echo 'case: yield counts a re-run attempt whose first attempt cannot be read as a full run' +reset_yield_fixtures +status_list "[$(full_run_success 100)]" +workflow_runs "[$(attempt_entry 4300 queued 2026-09-05T12:00:31Z 2 2026-09-05T12:05:02Z)]" +jobs_raw 4300 "$rerun_sibling_jobs" +printf '%s\n' 'gh: Internal Server Error (HTTP 500)' >"$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_4300_attempts_1_jobs.err" +run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true +expect_log '::error::superseded by full run https://github.com/melodic-software/ci-workflows/actions/runs/4300' +rm -f -- "$fixtures/GET_repos_melodic-software_ci-workflows_actions_runs_4300_attempts_1_jobs.err" + # Without yield overriding the wait, a 60-second ceiling would still poll. echo 'case: yield with nothing in flight reads the status once and never waits' reset_yield_fixtures diff --git a/README.md b/README.md index b49d20d..4bb20fc 100644 --- a/README.md +++ b/README.md @@ -330,18 +330,31 @@ consumer to audit it. - `yield-to-full-run: 'true'` and `timeout-minutes: 3` on the `ci-status` job. A contract-only run lists this workflow's in-flight runs on the head - SHA once, then reads the status once, and finishes in seconds. A full run + SHA, then reads the status, and finishes in seconds. A full run in flight supersedes it: it fails with `superseded by full run `, because that run's own `ci-status` check run is newer and decides the merge gate. With nothing in flight it is green on a recorded `success` and red otherwise. An in-flight sibling whose latest attempt skipped every job but - one, and that one has this run's gate name, is contract-only and ignored; - a full run whose lanes skipped before its gate job exists is not. Any + one, and that one has this run's gate name, is contract-only and ignored, + and so is a re-run attempt whose first attempt had that shape (a re-run + keeps its event, and the full run's own re-runs of failed contract-only + runs are such attempts); a full run whose lanes skipped before its gate + job exists is not. Any other sibling, including one whose jobs are not listed yet or cannot be read, counts as a full run, so a misread goes red, never green. The writer of a `success` already on the SHA is ignored while the attempt that wrote it finishes, which is what lets the - re-run it starts pass. It overrides `carry-forward-wait-seconds`, needs + re-run it starts pass. One state waits: when the newest status is a + `success` written by the current attempt of the one full run in flight, + and that run's aggregate step is still running (re-running failed + siblings, after its last listing of them), the run re-lists and re-reads + every 5 seconds for up to 30 until that step finishes. It then carries the + `success` once the step reads back `success`, so a contract-only run that + starts after the full run's sibling scan goes green instead of red with + nothing left to re-run it. A running step cannot prove it wrote the + status, so the run never carries it sooner. At 30 seconds it fails + superseded; a full run still waiting on it re-runs it once it completes. + It overrides `carry-forward-wait-seconds`, needs `actions: read`, and fails the run when the listing fails three times. (`carry-forward-wait-seconds: '0'` alone reads the status once with no listing, so it can carry an older `success` across a full run that is @@ -380,9 +393,10 @@ consumer to audit it. measured, so the full run's later `ci-status` supersedes the red. Only `success`, `skipped` and `neutral` satisfy a required check, so the red never passes on its own. The remaining gap: a contract-only run still in flight - when the full run's `rerun-wait-seconds` wait ends is not re-run, and a re-run - that reads the status before the full run's step finishes reads `pending`; - either one's message ends by saying to re-run it if it stays red. + when the full run's `rerun-wait-seconds` wait ends is not re-run, and without + `yield-to-full-run` a re-run that reads the status before the full run's step + finishes reads `pending`; either one's message ends by saying to re-run it if + it stays red. Permissions: the first job needs `statuses: write`; the `ci-status` job needs `statuses: write` and `actions: write` (which covers the `actions: read` From a22edd74a7cc43df4b202cd11d494440d91961be Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:04:58 -0400 Subject: [PATCH 2/4] fix(ci-status): log each failed run the sibling re-run scan considers The full run's re-run scan now names the failed runs it found and, for each one it passes over, why, so a scan that re-runs nothing shows what it saw. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../pr-require-checks/aggregate-results/run.sh | 2 ++ .../pr-require-checks/aggregate-results/run.test.sh | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/.github/actions/pr-require-checks/aggregate-results/run.sh b/.github/actions/pr-require-checks/aggregate-results/run.sh index 0368caf..42e6e42 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.sh @@ -1056,6 +1056,7 @@ rerun_failed_contract_only_siblings() { candidates="$(jq -r --argjson self "$GITHUB_RUN_ID" \ '[ .workflow_runs[]? | select(.status == "completed" and .conclusion == "failure" and (.id // $self) != $self) | .id ] | sort | join(" ")' \ <"$runs")" + echo "Failed run(s) of this workflow on ${SHA} to consider for a re-run: ${candidates:-none}." for id in $candidates; do # shellcheck disable=SC2310 # gh_api handles its own errexit; the caller classifies the status. if ! gh_api GET "repos/${REPOSITORY}/actions/runs/${id}/jobs?filter=latest&per_page=100"; then @@ -1064,6 +1065,7 @@ rerun_failed_contract_only_siblings() { fi contract_shaped="$(jq -r '[ .jobs[]? | .conclusion ] | ((map(select(. != "skipped")) == ["failure"]) and (length > 1))' <"$gh_stdout")" if [[ "$contract_shaped" != true ]]; then + echo "Not re-running failed run ${id} on ${SHA}: its latest attempt is not contract-only (more than its gate ran, or the gate did not fail)." continue fi # shellcheck disable=SC2310 # gh_api handles its own errexit; the caller classifies the status. diff --git a/.github/actions/pr-require-checks/aggregate-results/run.test.sh b/.github/actions/pr-require-checks/aggregate-results/run.test.sh index 11c40a9..896a3fe 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.test.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.test.sh @@ -1795,6 +1795,18 @@ run_case 0 'success success' pass false true YIELD_TO_FULL_RUN=true expect_log 'All lanes passed or were skipped.' expect_no_gh_call 'actions/' +# Without a line per failed run, a scan that re-runs nothing cannot show which +# runs it saw and why it passed over each. +echo 'case: a full-mode success logs each failed run it considered and why' +reset_yield_fixtures +workflow_runs "[$(completed_run 5000 failure),$(completed_run 5100 failure)]" +jobs_for 5000 failure skipped +jobs_for 5100 failure success +run_case 0 'success success' pass false true RERUN_CONTRACT_ONLY_SIBLINGS=true RERUN_WAIT_SECONDS=0 +expect_log "Failed run(s) of this workflow on ${sha} to consider for a re-run: 5000 5100." +expect_log "Re-running failed contract-only run 5000 on ${sha}." +expect_log "Not re-running failed run 5100 on ${sha}: its latest attempt is not contract-only" + # --- full mode: waiting for in-flight contract-only siblings ----------------- rerun_runs_count="$calls/${workflow_runs_key}.count" From 6093ca0fd721ca0b34c71dc431ff534e1bc2037d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:19:44 -0400 Subject: [PATCH 3/4] docs(ci-status): align re-run comments with the yield writer wait Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt --- .../actions/pr-require-checks/aggregate-results/run.sh | 8 +++++--- .../pr-require-checks/aggregate-results/run.test.sh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/pr-require-checks/aggregate-results/run.sh b/.github/actions/pr-require-checks/aggregate-results/run.sh index 42e6e42..08024cf 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.sh @@ -994,9 +994,11 @@ fi # `skipped`). A contract-only run only ever runs its gate. A sibling whose jobs # cannot be read is waited for, and so is a full run whose lanes are still # queued, for at most the ceiling. Re-runs are issued last, so this step -# usually finishes before a re-run's runner starts; a re-run that reads the -# status while this step is still running reads `pending` and goes red, as it -# could before. A sibling still in flight at the ceiling is not re-run, and its +# usually finishes before a re-run's runner starts. A re-run that reads the +# status while this step is still running waits for it under yield, up to +# YIELD_WRITER_WAIT_SECONDS, and passes once it succeeds; with a carry-forward +# wait of `0` it reads `pending` and goes red, as it could before. A sibling +# still in flight at the ceiling is not re-run, and its # message still ends by telling the reader to re-run it if it stays red. # # Nothing here changes this run's verdict: the success is already recorded, so diff --git a/.github/actions/pr-require-checks/aggregate-results/run.test.sh b/.github/actions/pr-require-checks/aggregate-results/run.test.sh index 896a3fe..e7b100b 100755 --- a/.github/actions/pr-require-checks/aggregate-results/run.test.sh +++ b/.github/actions/pr-require-checks/aggregate-results/run.test.sh @@ -1705,7 +1705,7 @@ run_case 1 'skipped skipped' pass true true YIELD_TO_FULL_RUN=true RERUN_CONTRAC expect_log "${fail_prefix}${absent_remedy}" expect_no_log 'Carried forward' -# provisioning run 37254877117: full run 4000 recorded success and, still in +# A contract-only run starts late: full run 4000 recorded success and, still in # its aggregate step, re-ran the failed contract-only run 4300, whose attempt 2 # lists only its queued gate so far. Without classifying a re-run attempt by # its first attempt, 4300 counts as a full run and this run fails superseded by From 0ee421e4335a91a0db820041df43c99bb4b9d0a5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:21:44 -0400 Subject: [PATCH 4/4] chore: re-run review lanes on a ready head Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01TRhsMPSTMFo99ik7gkdgQt