eval-browser-dispatch.yml

name: eval-browser-dispatch

# Runs `claude plugin eval` on a fresh GitHub-hosted runner, by hand only, for
# eval cases that drive a browser through Bash. A local run cannot: the eval's
# throwaway HOME hides tools installed under /home, and a symlinked ~/.docker
# makes the eval refuse any Bash-granting run. Here node, playwright-cli and
# Chromium are installed outside HOME and on PATH.
#
# Lane hardening follows ADR 0049
# (docs/adr/0049-run-ci-lanes-on-github-hosted-runners-under-trigger-and-token-hardening.md):
# the dispatcher must be on the trusted-actor list, the org kill switch is
# read before the model step, the job token can only read contents, and each
# dispatch gets a fresh runner. No write token exists in this job. The lane
# runs only when dispatched on the default branch, so every eval case it runs
# has merged through review.
#
# Only an env allowlist (PATH, ANTHROPIC_*, CLAUDE_CODE_*, EVAL_*) reaches an
# eval run (code.claude.com/docs/en/plugin-evals, "How runs are isolated", as
# of 2026-10-10), so PLAYWRIGHT_BROWSERS_PATH does not. A case's scaffold gets
# no EVAL_* variable either, so a case that needs the Chromium below pins it by
# path under /opt/pw (plugins/testing/evals/browser-probe/scaffold.sh).

on:
  workflow_dispatch:
    inputs:
      target:
        description: Eval target, a plugin directory or one case file.
        required: true
        type: string
        default: plugins/testing
      tag:
        description: Optional --tag filter; space-separated tags, any of which selects a case.
        required: false
        type: string
        default: ""
      runs:
        description: Runs per case per arm.
        required: true
        type: number
        default: 1
      allow-tools:
        description: Tools granted beyond the read-only set, comma-separated.
        required: true
        type: string
        default: Bash(playwright-cli *)
      debug-errors:
        description: Print short Bash error excerpts from each eval trace to the PUBLIC job log.
        required: false
        type: boolean
        default: false
      env-scrub:
        description: >-
          Strip credentials from Bash subprocesses in the eval
          (CLAUDE_CODE_SUBPROCESS_ENV_SCRUB). Turning it off lets Bash
          subprocesses in the eval see the Claude credentials; turn it off only
          for cases whose allowed tools cannot exfiltrate, such as Bash(echo *).
          What those tools print lands in the public results artifact and, with
          debug-errors, the public job log.
        required: false
        type: boolean
        default: true

permissions: {}

jobs:
  eval:
    # Default branch only: a dispatch on any other ref, or an event that names
    # no default branch, skips the job before any step runs.
    if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    permissions:
      contents: read # check out the repository; nothing in this job writes
    steps:
      # The dispatched commit, which the job if: holds to the default branch.
      - name: Check out the default branch
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: .base
          persist-credentials: false

      - name: Set up Node
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version-file: .base/.node-version
          package-manager-cache: false

      - name: Check the kill switch
        id: kill
        uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository]
        with:
          value: ${{ vars.CLAUDE_LANES_DISABLED }}
          repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }}

      - name: Stop when the kill switch is on
        if: steps.kill.outputs.proceed != 'true'
        env:
          REASON: ${{ steps.kill.outputs.reason }}
        run: |
          echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})"
          exit 1

      # check-trusted-trigger needs a PR, and a dispatch here names none, so
      # the dispatcher is matched against the same list directly: the event
      # actor by numeric id, the re-runner by login, as that action does. Both
      # must be listed humans, and the lanes App bot is denied by id, as in
      # pr-run-activity-write.yml.
      - name: Check the dispatcher is a trusted human
        env:
          LIST: .base/.github/standards/trusted-actors/trusted-actors.json
          ACTOR_ID: ${{ github.actor_id }}
          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
          APP_SENDER_ID: ${{ vars.AUTOMATION_LANES_APP_SENDER_ID }}
        run: |
          if ! [[ $APP_SENDER_ID =~ ^[1-9][0-9]*$ ]]; then
            echo "::error::the AUTOMATION_LANES_APP_SENDER_ID variable must hold the lanes App bot's numeric user id"
            exit 1
          fi
          if [ "$ACTOR_ID" = "$APP_SENDER_ID" ]; then
            echo "::error::bot-actor: the lanes App bot cannot dispatch this lane"
            exit 1
          fi
          if ! jq -e --argjson id "$ACTOR_ID" --argjson app "$APP_SENDER_ID" --arg login "$TRIGGERING_ACTOR" '
              .version == 1
              and any(.actors[]; .id == $id and .kind == "human")
              and any(.actors[];
                (.login | ascii_downcase) == ($login | ascii_downcase)
                and .kind == "human" and .id != $app)
            ' "$LIST" >/dev/null; then
            echo "::error::untrusted-actor: the dispatcher or re-runner is not a listed human"
            exit 1
          fi

      # The tools and Chromium go under /opt, outside HOME, which each eval
      # run replaces with a temporary one. Only these directories are made
      # while sudo works; no third-party code runs before isolation.
      - name: Create the tool directories
        run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools /opt/pw

      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
      # and docker access for the rest of the job.
      - name: Isolate the runner
        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
        with:
          install-subprocess-isolation: "true"

      # Every binary this job runs comes from the locked root toolchain. Its
      # install scripts run here, after sudo and docker access are gone.
      - name: Install the locked root toolchain
        run: |
          cp .base/package.json .base/package-lock.json /opt/tools/
          npm ci --prefix /opt/tools
          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"

      # Chromium comes from the playwright that the locked playwright-cli
      # uses. Without sudo there is no --with-deps: the hosted image already
      # carries a browser's system libraries.
      - name: Install Chromium
        env:
          PLAYWRIGHT_BROWSERS_PATH: /opt/pw
        run: |
          node /opt/tools/node_modules/playwright/cli.js install chromium
          command -v playwright-cli claude

      # Every Bash call in an eval run goes through Claude Code's bubblewrap
      # sandbox (code.claude.com/docs/en/sandboxing, "Set up Linux and WSL2",
      # as of 2026-10-10). Starting bwrap here as the runner user, with the
      # whole filesystem (so /opt/tools and the setup-node toolcache) bound
      # read-only, fails the job with bwrap's own error if bwrap cannot start
      # a sandbox, before any eval spends tokens. Its flags are not Claude
      # Code's own, so a pass does not prove the full sandbox starts.
      - name: Check bwrap starts a sandbox
        run: |
          fail() {
            local msg=${2//'%'/'%25'}
            msg=${msg//$'\r'/'%0D'}
            echo "::error::$1: ${msg//$'\n'/'%0A'}"
            exit 1
          }
          for tool in bwrap socat; do
            command -v "$tool" || fail "$tool is not on PATH" "the Bash sandbox needs it"
          done
          sysctl kernel.apparmor_restrict_unprivileged_userns || true
          sandbox=(bwrap --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --unshare-all --die-with-parent)
          if ! out=$("${sandbox[@]}" true 2>&1); then
            fail "bwrap cannot start a sandbox as the runner user" "$out"
          fi
          if ! out=$("${sandbox[@]}" playwright-cli --version 2>&1); then
            fail "playwright-cli does not start inside bwrap" "$out"
          fi
          echo "playwright-cli inside bwrap: $out"

      - name: Record that env scrub is off
        if: ${{ !inputs.env-scrub }}
        env:
          DISPATCHER: ${{ github.actor }}
        run: echo "::warning::CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is off for this run; Bash subprocesses can see the Claude credentials. Dispatched by ${DISPATCHER}."

      - name: Run the plugin eval
        working-directory: .base
        env:
          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          # Strips credentials from Bash subprocesses inside each eval run.
          # The env-scrub input set false gives "0", the opt-out value Claude
          # Code's own messages name.
          CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: ${{ inputs.env-scrub && '1' || '0' }}
          TARGET: ${{ inputs.target }}
          TAGS: ${{ inputs.tag }}
          RUNS: ${{ inputs.runs }}
          ALLOW_TOOLS: ${{ inputs.allow-tools }}
          OUT: ${{ runner.temp }}/eval-results
          DEBUG_ERRORS: ${{ inputs.debug-errors }}
        run: |
          case "$TARGET" in
            '' | -*) echo "::error::target must be a path"; exit 1 ;;
          esac
          if ! [[ $RUNS =~ ^[1-9][0-9]*$ ]]; then
            echo "::error::runs must be a positive integer"
            exit 1
          fi
          mkdir -p "$OUT"
          IFS=, read -ra tools <<< "$ALLOW_TOOLS"
          read -ra tags <<< "$TAGS"
          for value in "${tools[@]}" "${tags[@]}"; do
            case "$value" in
              -*) echo "::error::a tag or tool must not start with -"; exit 1 ;;
            esac
          done
          args=(
            "$TARGET"
            --trust-plugin
            --scaffold
            --no-publish
            --runs "$RUNS"
            --model claude-sonnet-5-5
            --judge-model claude-opus-5-5
            --max-cost-usd 5
            --output-dir "$OUT"
            --json "$OUT/results.json"
            --allow-tools "${tools[@]}"
          )
          if [ "${#tags[@]}" -gt 0 ]; then
            args+=(--tag "${tags[@]}")
          fi
          # Without --keep-temp each run's directory, trace.jsonl included,
          # is deleted when the run ends. It stays on the runner, outside $OUT.
          if [ "$DEBUG_ERRORS" = true ]; then
            args+=(--keep-temp)
          fi
          claude plugin eval "${args[@]}"

      # Opt-in because the job log of this public repository is public. Prints,
      # per run, at most 20 Bash calls whose tool_result has is_error true: the
      # command cut to 300 characters, and the result text cut to its first 120
      # and last 2500 characters when longer than 2620 (the end holds the daemon
      # log), each JSON-quoted onto one line. A second line, `key:`, holds the lines of the
      # full result that match FATAL, Check failed, ERROR:, sandbox, permission or errno
      # patterns (the cause line that a long crash stack pushes out of the tail), minus the
      # known .bashrc noise, de-duplicated, at most 15, each cut to 300 characters; it is
      # omitted when nothing matches. Nothing else from the trace is printed. Trace text is
      # tool output, so workflow commands are off while it prints.
      - name: Print Bash error excerpts
        if: always() && inputs.debug-errors
        env:
          RESULTS: ${{ runner.temp }}/eval-results/results.json
        run: |
          if [ ! -f "$RESULTS" ]; then
            echo "no results.json, so no traces to read"
            exit 0
          fi
          token=$(openssl rand -hex 16)
          echo "::stop-commands::$token"
          jq -r '.cases[]? as $c | $c.arms // {} | to_entries[] | .key as $arm
            | .value // [] | to_entries[]
            | [($c.name // $c.dir // "case"), $arm, (.key + 1 | tostring), (.value.tracePath // "")]
            | @tsv' "$RESULTS" |
            while IFS=$'\t' read -r name arm run trace; do
              echo "== $name, $arm arm, run $run"
              if [ ! -f "$trace" ]; then
                echo "   trace not found"
                continue
              fi
              jq -nrR '
                def tail_cut: if length > 2620 then .[:120] + " … " + .[-2500:] else . end;
                def key_lines:
                  [split("\n")[]
                    | select(test("FATAL|Check failed|ERROR:|No usable sandbox|Permission denied|Operation not permitted|EACCES|EPERM|ENOENT|errno")
                             and (contains(".bashrc: Permission denied") | not))
                    | .[:300]]
                  | reduce .[] as $l ([]; if index([$l]) then . else . + [$l] end)
                  | .[:15] | join("\n");
                [inputs | fromjson? | objects] as $m
                | ([$m[] | select(.type == "assistant") | .message.content[]? | objects
                    | select(.type == "tool_use" and .name == "Bash" and (.id | type) == "string")
                    | {key: .id, value: (.input.command // "" | tostring)}] | from_entries) as $bash
                | [$m[] | select(.type == "user") | .message.content[]? | objects
                    | select(.type == "tool_result" and .is_error == true)
                    | (.tool_use_id // "" | tostring) as $id
                    | select($bash | has($id))
                    | {cmd: $bash[$id],
                       out: (.content | if type == "array" then map(objects | select(.type == "text") | .text | tostring) | join("\n") else tostring end)}]
                | "   \(length) Bash error(s), showing up to 20",
                  (.[:20][] | "   $ \(.cmd[:300] | tojson)\n     -> \(.out | tail_cut | tojson)\(.out | key_lines | if . == "" then "" else "\n     key: \(tojson)" end)")
              ' "$trace" || echo "   could not read the trace"
            done || echo "could not read results.json"
          echo "::$token::"

      - name: Upload the eval results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ runner.temp }}/eval-results/
          if-no-files-found: warn
          retention-days: 30