diff --git a/.agents/skills/switchyard-stage-router-scorer/SKILL.md b/.agents/skills/switchyard-stage-router-scorer/SKILL.md new file mode 100644 index 00000000..01a6718f --- /dev/null +++ b/.agents/skills/switchyard-stage-router-scorer/SKILL.md @@ -0,0 +1,94 @@ +# Skill: switchyard-stage-router-scorer + +**description**: Score benchmark run trajectories through the stage-router Rust scorer and picker, then visualise score distributions. Use when you want to replay trajectories through the picker, analyse routing splits, or compare score distributions across configs. + +## Scripts + +| Script | Purpose | Input | Output | +|--------|---------|-------|--------| +| `benchmark/score_staged_run.py` | Score a live run dir via real picker | run dir path | per-turn JSONL + per-task CSV | + +## Quick Reference + +```bash +# Score a run +uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ +# → /tmp/-scores.jsonl (per turn) +# → /tmp/-per-task.csv (per task) + +# Custom threshold or window +uv run python benchmark/score_staged_run.py \ + --run benchmark/tb_runs/ \ + --threshold 0.15 --window 3 +``` + +## Scoring pipeline (what score_staged_run.py does per turn) + +``` +trajectory step (tool_use + tool_result) + → append to cumulative Anthropic messages list + → ChatRequest.anthropic({"model": ..., "messages": messages}) # Rust binding + → dc.process(ctx, request) # DimensionCollector — one per task, accumulates state + → get_tool_result_signal(ctx) # read signal from ctx + → stage_score_signal(signal) # raw (score, confidence) from the Rust scorer (for analysis) + → pick_capable_first(ctx, threshold) # actual cf decision (same ctx, no re-process) + → pick_efficient_first(ctx, threshold) # actual ef decision (same ctx, no re-process) +``` + +**Key:** `dc.process()` is called **once per turn** on a single ctx. Both pickers read the signal +already stored in that ctx — no duplicate processing. + +**What the picker does beyond raw score:** +- **escalate** (`should_escalate`): `compacted` OR `severity >= 1.0` → force CAPABLE +- **de-escalate** (`should_deescalate`): `tests_passed AND recent_write+edit >= 1 AND severity <= 0` → force EFFICIENT +- `confidence < threshold` → fall_open to default tier (CAPABLE for cf, EFFICIENT for ef) +- Only when `confidence >= threshold`: route by score direction + +## Per-turn JSONL schema + +```json +{ + "task_name": "terminal-bench/...", "trial_name": "...", "run_id": "...", + "reward": 1.0, "turn_depth": 5, + "score": -0.83, "confidence": 0.95, + "tool_name": "Bash", "is_error": false, + "write_count": 2, "edit_count": 1, "read_count": 3, + "no_error_streak": 4, "pure_bash_streak": 2, "tests_passed": false, + "pick_cf": 1, + "pick_ef": 0 +} +``` + +`pick_cf` / `pick_ef`: `1` = CAPABLE (Opus), `0` = EFFICIENT (Nemotron) + +## Per-task CSV columns + +`run_id, task_name, reward, n_turns, mean_score, mean_confidence, +pct_strong_clear, pct_strong_uncertain, pct_weak_uncertain, pct_weak_clear, +opus_pct_cf, nemotron_pct_cf, opus_pct_ef, nemotron_pct_ef` + +`opus_pct_cf` / `opus_pct_ef` are derived from actual `pick_cf` / `pick_ef` decisions, not score bands. + +## Band definitions (for histogram colouring, threshold T) + +| Band | Score range | cf default | ef default | +|------|-------------|------------|------------| +| strong_clear | ≥ T | Opus | Opus | +| strong_uncertain | (0, T) | Opus (fall_open) | Nemotron (fall_open) | +| weak_uncertain | (-T, 0) | Nemotron (fall_open) | Nemotron (fall_open) | +| weak_clear | ≤ -T | Nemotron | Nemotron | + +Overrides can change any band. Use `pick_cf`/`pick_ef` for the true decision. + +## Key findings (v0.2.0 baseline tarballs, T=0.20) + +- Distribution is **highly bimodal** — most turns land in strong_clear or weak_clear +- `cf, t=0.20`: ~34% Opus on partial run; ~42% on full baseline +- `ef, t=0.20`: ~20% Opus on partial run; ~41% on full baseline +- Live routing split differs from shadow score — Nemotron trajectories are shorter and reshape the distribution + +## Anti-patterns + +- Don't call `dc.process()` multiple times per turn for different pickers — both pickers read from the same ctx. One `dc.process()` call per turn is correct. +- Don't infer routing split from score bands alone — overrides and fall_open change the actual decision. Always use `pick_cf` / `pick_ef`. +- Don't compare cost directly across configs: Nemotron has ~39% cache hit rate vs ~92% for Opus. diff --git a/benchmark/calibration/stage_router/calibrate.py b/benchmark/calibration/stage_router/calibrate.py deleted file mode 100644 index 7b8a97d6..00000000 --- a/benchmark/calibration/stage_router/calibrate.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Calibrate stage_router confidence threshold from Harbor run outputs. - -Pass the Harbor run output directories for the pure-capable and pure-efficient arms: - - python calibrate.py --capable-run-dir /tmp/runs/capable --efficient-run-dir /tmp/runs/efficient - python sweep.py -""" -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -from signal_extractor import replay_trajectory, task_summary # noqa: E402 - -OUT = Path(__file__).parent - -_RANK = {"pass": 2, "fail": 1, "err": 0} - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Calibrate stage_router confidence threshold from Harbor run outputs." - ) - parser.add_argument( - "--capable-run-dir", - type=Path, - required=True, - help="Harbor output directory for the pure-capable run.", - ) - parser.add_argument( - "--efficient-run-dir", - type=Path, - required=True, - help="Harbor output directory for the pure-efficient probe run.", - ) - return parser.parse_args(argv) - - -def _outcome(result: dict) -> str: - if result.get("exception_info") is not None: - return "err" - reward = (result.get("verifier_result") or {}).get("rewards", {}).get("reward") - return "pass" if reward == 1.0 else "fail" - - -def read_arm(arm_dir: Path): - if not arm_dir.is_dir(): - raise FileNotFoundError(f"{arm_dir} is not a directory") - - outcomes: dict[str, str] = {} - feats: dict[str, dict] = {} - per_turn: dict[str, list] = {} - - for td in sorted(arm_dir.iterdir()): - if not td.is_dir(): - continue - rp = td / "result.json" - if not rp.exists(): - continue - result = json.loads(rp.read_text()) - task = result.get("task_name") or td.name - outcome = _outcome(result) - trajs = list(td.glob("agent/sessions/projects/-app/*.jsonl")) - signals = replay_trajectory(trajs[0]) if trajs else [] - - if _RANK[outcome] > _RANK.get(outcomes.get(task, ""), -1): - outcomes[task] = outcome - feats[task] = task_summary(signals) - per_turn[task] = [ - { - "task": task, - "turn": i, - "severity": s.severity, - "write_count": s.write_count, - "edit_count": s.edit_count, - "read_count": s.read_count, - "turn_depth": s.turn_depth, - "pure_bash_streak": s.pure_bash_streak, - "tests_passed": s.tests_passed, - } - for i, s in enumerate(signals) - ] - return outcomes, feats, per_turn - - -def main(argv: list[str] | None = None): - args = parse_args(argv) - arms = { - "capable": args.capable_run_dir, - "efficient": args.efficient_run_dir, - } - all_outcomes: dict[str, dict] = {} - all_feats: dict[str, dict] = {} - all_turns: dict[str, dict] = {} - - for arm, d in arms.items(): - outcomes, feats, turns = read_arm(d) - all_outcomes[arm] = outcomes - all_feats[arm] = feats - all_turns[arm] = turns - passes = sum(1 for v in outcomes.values() if v == "pass") - print(f"{arm}: {len(outcomes)} tasks pass={passes}") - - arm_names = list(arms) - if len(arm_names) >= 2: - s_out = all_outcomes[arm_names[0]] - w_out = all_outcomes[arm_names[1]] - buckets: dict[str, int] = {"RESCUE": 0, "LOSS": 0, "SAFE": 0, "HARD": 0} - for t in set(s_out) | set(w_out): - s, w = s_out.get(t), w_out.get(t) - if s == "pass" and w == "pass": - buckets["SAFE"] += 1 - elif s == "fail" and w == "pass": - buckets["RESCUE"] += 1 - elif s == "pass" and w != "pass": - buckets["LOSS"] += 1 - elif s == "fail" and w != "pass": - buckets["HARD"] += 1 - print() - for b, n in buckets.items(): - print(f" {b}: {n}") - - with (OUT / "per_task.jsonl").open("w") as f: - for arm in arm_names: - for task, outcome in all_outcomes[arm].items(): - f.write(json.dumps({"task": task, "arm": arm, "outcome": outcome, - **all_feats[arm].get(task, {})}) + "\n") - - with (OUT / "per_turn.jsonl").open("w") as f: - for arm in arm_names: - for _task, turn_list in all_turns[arm].items(): - for rec in turn_list: - f.write(json.dumps({**rec, "arm": arm}) + "\n") - - print(f"\nWrote per_task.jsonl + per_turn.jsonl → {OUT}") - print("Run sweep.py to score escalation policies.") - - -if __name__ == "__main__": - main() diff --git a/benchmark/calibration/stage_router/signal_extractor.py b/benchmark/calibration/stage_router/signal_extractor.py deleted file mode 100644 index 007d1311..00000000 --- a/benchmark/calibration/stage_router/signal_extractor.py +++ /dev/null @@ -1,297 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python port of crates/switchyard-components/src/dimension_collector/tool_signals.rs. - -Reads a claude-code session JSONL trajectory and reconstructs the per-turn -ToolResultSignal that the stage_router picker would have seen. Output is one record -per assistant turn. -""" -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -SOFT, HARD, CRITICAL = 0.3, 0.7, 1.0 - -ERROR_PATTERNS: list[tuple[str, float, tuple[str, ...]]] = [ - ("oom", CRITICAL, ("out of memory", "memoryerror", "cannot allocate memory")), - ("connection_refused", CRITICAL, ("connection refused", "connectionrefusederror", "econnrefused")), - ("traceback", HARD, ("traceback (most recent call last)",)), - ("import_error", HARD, ("modulenotfounderror:", "importerror:", "no module named ")), - ("cmd_not_found", HARD, ("command not found", "not found\n", "/usr/bin/env: ")), - ("assertion", HARD, ("assertionerror",)), - ("value_error", HARD, ("valueerror:",)), - ("syntax_error", HARD, ("syntaxerror:",)), - ("timeout", HARD, ("timed out", "timeouterror", "timeout expired", "deadline exceeded")), - ("no_such_file", HARD, ("filenotfounderror:", "no such file or directory")), - ("exit_nonzero", SOFT, ("exit code 1", "exit code 2", "exit status 1", "returned non-zero", "exited with code")), -] - -EDIT_TOOL_NAMES = {"edit", "multiedit", "notebookedit", "str_replace", "str_replace_based_edit_tool", "text_editor"} -WRITE_TOOL_NAMES = {"write", "create_file", "new_file"} -READ_TOOL_NAMES = {"read", "view"} -PLAN_TOOL_NAMES = {"todowrite", "todo_write", "todo", "update_plan"} -BASH_TOOL_NAMES = {"bash", "shell_command", "shell", "local_shell_call"} - -BASH_WRITE_PATTERNS = ("cat >", "cat >>", "echo >", "echo >>", "tee ", "printf >", "printf >>", "> /", ">> /", "<< 'eof'", "< tuple[float, list[str]]: - lower = text.lower() - patterns: list[str] = [] - severity = 0.0 - for name, sev, subs in ERROR_PATTERNS: - if any(s in lower for s in subs): - patterns.append(name) - if sev > severity: - severity = sev - return severity, patterns - - -def classify_tool_call(name: str, command: str | None) -> str: - lower = name.lower() - if lower in WRITE_TOOL_NAMES: - return "Write" - if lower in EDIT_TOOL_NAMES: - return "Edit" - if lower in READ_TOOL_NAMES: - return "Read" - if lower in PLAN_TOOL_NAMES: - return "Plan" - if lower in BASH_TOOL_NAMES and command is not None: - if any(p in command for p in BASH_WRITE_PATTERNS): - return "Write" - if any(p in command for p in BASH_EDIT_PATTERNS): - return "Edit" - if any(p in command for p in BASH_READ_PATTERNS): - return "Read" - return "Other" - - -def _detect_tests_passed(tool_texts: list[str]) -> bool: - recent = tool_texts[-3:] if len(tool_texts) > 3 else tool_texts - for text in recent: - lower = text.lower() - if any(p in lower for p in TEST_PASS_PHRASES) and not any(p in lower for p in TEST_FAILURE_PHRASES): - return True - return False - - -def _compute_no_error_streak(tool_texts: list[str]) -> int: - streak = 0 - for text in reversed(tool_texts): - sev, _ = classify_text(text) - if sev > 0.0: - break - streak += 1 - return streak - - -def _build_signal(tool_texts: list[str], tool_calls: list[tuple[str, str | None]], turn_depth: int, prompt_char_count: int) -> ToolResultSignal: - severity, patterns = classify_text(tool_texts[-1]) if tool_texts else (0.0, []) - no_error_streak = _compute_no_error_streak(tool_texts) - - recent_start = max(0, len(tool_calls) - RECENT_WINDOW) - write_count = edit_count = read_count = todowrite_count = 0 - recent_write_count = recent_edit_count = recent_read_count = recent_todowrite_count = 0 - pure_bash_streak = 0 - streak_open = True - - for i in range(len(tool_calls) - 1, -1, -1): - name, cmd = tool_calls[i] - cat = classify_tool_call(name, cmd) - if streak_open: - if cat == "Other": - pure_bash_streak += 1 - else: - streak_open = False - in_recent = i >= recent_start - if cat == "Write": - write_count += 1 - if in_recent: - recent_write_count += 1 - elif cat == "Edit": - edit_count += 1 - if in_recent: - recent_edit_count += 1 - elif cat == "Read": - read_count += 1 - if in_recent: - recent_read_count += 1 - elif cat == "Plan": - todowrite_count += 1 - if in_recent: - recent_todowrite_count += 1 - - return ToolResultSignal( - severity=severity, - patterns=patterns, - no_error_streak=no_error_streak, - edit_count=edit_count, - write_count=write_count, - read_count=read_count, - todowrite_count=todowrite_count, - recent_edit_count=recent_edit_count, - recent_write_count=recent_write_count, - recent_read_count=recent_read_count, - recent_todowrite_count=recent_todowrite_count, - pure_bash_streak=pure_bash_streak, - tests_passed=_detect_tests_passed(tool_texts), - turn_depth=turn_depth, - prompt_char_count=prompt_char_count, - ) - - -def _content_to_text(content: Any) -> str | None: - if isinstance(content, str): - return content - if isinstance(content, list): - parts = [] - for b in content: - if isinstance(b, dict) and b.get("type") == "text" and isinstance(b.get("text"), str): - parts.append(b["text"]) - return "\n".join(parts) if parts else None - return None - - -def replay_trajectory(jsonl_path: Path) -> list[ToolResultSignal]: - """Walk a claude-code session JSONL and emit per-assistant-turn signals. - - Mirrors `extract_from_messages_anthropic` in tool_signals.rs: each assistant - turn produces a signal computed from all messages up to (but not including) - the assistant turn itself — i.e. what the picker would have seen as input - when deciding which model to call for that turn. - """ - tool_texts: list[str] = [] - tool_calls: list[tuple[str, str | None]] = [] - prompt_char_count = 0 - msg_count = 0 # messages-so-far (proxy for turn_depth in the Rust impl) - signals: list[ToolResultSignal] = [] - - with open(jsonl_path, encoding="utf-8", errors="replace") as f: - for line in f: - try: - ev = json.loads(line) - except json.JSONDecodeError: - continue - et = ev.get("type") - if et not in ("user", "assistant"): - continue - - if et == "assistant": - # Snapshot signal before processing the assistant content — this - # is what the picker would have seen when deciding which model - # serves THIS assistant turn. - signals.append(_build_signal(tool_texts.copy(), tool_calls.copy(), msg_count, prompt_char_count)) - - msg = ev.get("message", {}) - content = msg.get("content") - msg_count += 1 - - if et == "user": - # Claude-code stores user content as either a plain string - # (initial prompt or text follow-up) or a list with - # tool_result / text blocks. - if isinstance(content, str): - prompt_char_count = len(content) - elif isinstance(content, list): - for b in content: - if not isinstance(b, dict): - continue - bt = b.get("type") - if bt == "tool_result": - text = _content_to_text(b.get("content")) - if text is not None: - tool_texts.append(text) - elif bt == "text": - t = b.get("text") - if isinstance(t, str): - prompt_char_count = len(t) - elif et == "assistant": - if isinstance(content, list): - for b in content: - if not isinstance(b, dict): - continue - if b.get("type") == "tool_use": - name = b.get("name") - inp = b.get("input") - cmd = None - if isinstance(inp, dict): - c = inp.get("command") - if isinstance(c, str): - cmd = c.lower() - if isinstance(name, str): - tool_calls.append((name, cmd)) - - return signals - - -def task_summary(signals: list[ToolResultSignal]) -> dict[str, Any]: - """Aggregate per-turn signals into one feature vector per task.""" - if not signals: - return { - "turns": 0, - "max_severity": 0.0, - "max_turn_depth": 0, - "max_pure_bash": 0, - "total_writes": 0, - "total_edits": 0, - "total_reads": 0, - "total_todowrites": 0, - "max_recent_reads": 0, - "max_recent_writes": 0, - "max_recent_todowrites": 0, - "prompt_char_count": 0, - "ever_tests_passed": False, - "ever_severity_critical": False, - "ever_severity_hard": False, - "ever_severity_soft": False, - } - last = signals[-1] - return { - "turns": len(signals), - "max_severity": max(s.severity for s in signals), - "max_turn_depth": max(s.turn_depth for s in signals), - "max_pure_bash": max(s.pure_bash_streak for s in signals), - "total_writes": last.write_count, - "total_edits": last.edit_count, - "total_reads": last.read_count, - "total_todowrites": last.todowrite_count, - "max_recent_reads": max(s.recent_read_count for s in signals), - "max_recent_writes": max(s.recent_write_count for s in signals), - "max_recent_todowrites": max(s.recent_todowrite_count for s in signals), - "prompt_char_count": signals[0].prompt_char_count if signals else 0, - "ever_tests_passed": any(s.tests_passed for s in signals), - "ever_severity_critical": any(s.severity >= CRITICAL for s in signals), - "ever_severity_hard": any(s.severity >= HARD for s in signals), - "ever_severity_soft": any(s.severity >= SOFT for s in signals), - } diff --git a/benchmark/calibration/stage_router/sweep.py b/benchmark/calibration/stage_router/sweep.py deleted file mode 100644 index 11950c21..00000000 --- a/benchmark/calibration/stage_router/sweep.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Score escalation policies against per_turn.jsonl + per_task.jsonl. - -Run calibrate.py first to generate the input files. -""" -from __future__ import annotations - -import json -from collections import defaultdict -from pathlib import Path - -OUT = Path(__file__).parent - - -def load(): - by: dict = defaultdict(list) - with (OUT / "per_turn.jsonl").open() as f: - for line in f: - r = json.loads(line) - by[(r["task"], r["arm"])].append(r) - for k in by: - by[k].sort(key=lambda r: r["turn"]) - - outcomes: dict = defaultdict(dict) - rank = {"pass": 2, "fail": 1, "err": 0} - with (OUT / "per_task.jsonl").open() as f: - for line in f: - r = json.loads(line) - cur = outcomes[r["task"]].get(r["arm"]) - if cur is None or rank[r["outcome"]] > rank[cur]: - outcomes[r["task"]][r["arm"]] = r["outcome"] - return by, outcomes - - -def score(by, outcomes, picker_fn, capable="capable", efficient="efficient"): - P = F = E = esc = 0 - for task, arms in outcomes.items(): - turns = by.get((task, capable), []) - escalate = picker_fn(turns) if turns else False - o = arms.get(efficient if escalate else capable, "err") - if escalate: - esc += 1 - if o == "pass": - P += 1 - elif o == "fail": - F += 1 - else: - E += 1 - total = P + F - return { - "pass_count": P, - "pct": P / max(1, total) * 100, - "esc_rate": esc / max(1, len(outcomes)), - } - - -# --- Escalation policies --- - -def always_stay(turns): return False -def always_escalate(turns): return True - - -def no_write_by_turn(N: int): - def p(turns): - for t in turns: - if t["turn_depth"] >= N: - return t["write_count"] == 0 and t["edit_count"] == 0 - return False - return p - - -def silent_stall(td: int, max_reads: int = 4): - def p(turns): - return any( - t["turn_depth"] >= td and t["write_count"] == 0 - and t["edit_count"] == 0 and t["read_count"] <= max_reads - for t in turns - ) - return p - - -def combo(turns): - """Escalate on stall or critical error; stay if tests passed.""" - for t in turns: - if t["tests_passed"]: - return False - if t["severity"] >= 1.0: - return True - if t["turn_depth"] >= 8 and t["write_count"] == 0 and t["edit_count"] == 0: - return True - if t["turn_depth"] >= 7 and t["severity"] >= 0.7 and t["write_count"] == 0: - return True - if t["pure_bash_streak"] >= 4 and t["write_count"] == 0: - return True - return False - - -def main(): - by, outcomes = load() - policies = [ - ("always_stay", always_stay), - ("always_escalate", always_escalate), - ("no_write_by_turn=8", no_write_by_turn(8)), - ("no_write_by_turn=12", no_write_by_turn(12)), - ("no_write_by_turn=15", no_write_by_turn(15)), - ("silent_stall td>=8 R<=2", silent_stall(8, 2)), - ("silent_stall td>=8 R<=4", silent_stall(8, 4)), - ("silent_stall td>=12 R<=6", silent_stall(12, 6)), - ("combo", combo), - ] - print(f"{'policy':40} pass% esc%") - print("-" * 55) - for name, p in policies: - r = score(by, outcomes, p) - print(f" {name:38} {r['pct']:5.1f}% {r['esc_rate']*100:4.0f}%") - - -if __name__ == "__main__": - main() diff --git a/benchmark/dimension_calibration/corpus_manifest.yaml b/benchmark/dimension_calibration/corpus_manifest.yaml deleted file mode 100644 index a49ce7b5..00000000 --- a/benchmark/dimension_calibration/corpus_manifest.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Calibration corpus for the DimensionCollector. Each entry points at one -# Harbor run; the extractor turns that run into a normalized stream of -# ScoredPrompt records the calibration runner consumes. -# -# Paths are RELATIVE TO THE SWITCHYARD REPO ROOT. The runner resolves them -# against the repo so the same manifest works on any machine that has the -# referenced Harbor runs in the same place. -# -# Switchyard writes Harbor TBLite runs under benchmark/tb_runs/ (gitignored) -# via `switchyard verify --route-profile --model `. Each run -# lands at: -# -# benchmark/tb_runs//jobs//__/agent/ -# episode-0/prompt.txt -# episode-1/prompt.txt -# ... -# -# Point each entry below at the inner `jobs//` directory. -# -# To add your own run: -# 1. produce a Harbor run (`switchyard verify ...`) -# 2. add an entry below with a unique `id`, a meaningful `source` label, -# and the path to its `jobs//` directory -# 3. `uv run python benchmark/dimension_calibration/run.py` -# -# The per-dimension report under report/ is overwritten on every run. - -runs: - # Example entries — replace the paths with your local Harbor run directories. - # The runner skips entries whose path does not exist, so unused stubs are - # harmless. - - - id: stage_router-strong-default-deepseek - extractor: harbor - source: stage_router-strong-default-deepseek - path: "benchmark/tb_runs/tb-lite-stage_router-strong-default-deepseek/jobs/tb-lite-stage_router-strong-default-deepseek" - label: "StageRouter — capable_first picker, Opus + DeepSeek" - - - id: stage_router-strong-default-nemotron - extractor: harbor - source: stage_router-strong-default-nemotron - path: "benchmark/tb_runs/tb-lite-stage_router-strong-default-nemotron/jobs/tb-lite-stage_router-strong-default-nemotron" - label: "StageRouter — capable_first picker, Opus + Nemotron" - - - id: all-opus - extractor: harbor - source: all-opus - path: "benchmark/tb_runs/tb-lite-single-opus-4-7/jobs/tb-lite-single-opus-4-7" - label: "All-Opus baseline" - - - id: all-deepseek - extractor: harbor - source: all-deepseek - path: "benchmark/tb_runs/tb-lite-single-deepseek-v4-pro/jobs/tb-lite-single-deepseek-v4-pro" - label: "All-DeepSeek baseline" diff --git a/benchmark/dimension_calibration/extractors/__init__.py b/benchmark/dimension_calibration/extractors/__init__.py deleted file mode 100644 index dc0417c9..00000000 --- a/benchmark/dimension_calibration/extractors/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Trajectory-to-prompt extractors for the dimension-collector calibration corpus. - -Each extractor turns one harness run (currently: Harbor episode dirs) into a -stream of :class:`ScoredPrompt` records the calibration runner consumes. Two -scoring scopes per prompt: - -* ``full`` — the concatenated message blob the proxy actually sees in - production. System prompt, prior tool results, and tool calls all in. -* ``latest`` — the most recent user-role text content only. Cleaner signal - isolation; useful as a counterpoint to ``full``. -""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ScoredPrompt: - """Normalized prompt record consumed by the calibration runner. - - Fields are designed to be report-sliceable downstream: every analytical - cut in the report (per-source, per-run, per-turn-position) reads from - these columns and nothing else. - """ - - run_id: str - source: str # label from the manifest entry (e.g. "stage_router-settle-aware") - task: str # task id (e.g. "book-portfolio-analysis") - turn_idx: int # 0-based turn within the trajectory - full: str # full prompt blob (system + history + latest user) - latest: str # latest user-role text content only diff --git a/benchmark/dimension_calibration/extractors/harbor.py b/benchmark/dimension_calibration/extractors/harbor.py deleted file mode 100644 index 828d6e6a..00000000 --- a/benchmark/dimension_calibration/extractors/harbor.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Harbor episode dirs → ScoredPrompt stream. - -Harbor lays out one directory per task with an ``agent/`` subtree:: - - / - __/ - agent/ - episode-0/prompt.txt ← LLM-inbound prompt for turn 0 - episode-1/prompt.txt - ... - -Switchyard's ``switchyard verify --route-profile ...`` writes Harbor TBLite -runs at ``benchmark/tb_runs//jobs//`` — that's the -directory to point the manifest at. - -The ``source`` field on emitted records comes from the manifest entry's -``source:`` key (defaults to ``"harbor"``). Use distinct source labels on -different runs (e.g. ``stage_router-settle-aware``, ``all-opus``) so the by-source slice in -the report compares like-with-like. -""" - -from __future__ import annotations - -import re -from collections.abc import Iterator -from pathlib import Path - -from . import ScoredPrompt - -_EPISODE_DIR_PATTERN = re.compile(r"^episode-(\d+)$") -DEFAULT_SOURCE = "harbor" - - -def extract( - run_id: str, - run_root: Path, - *, - source: str = DEFAULT_SOURCE, -) -> Iterator[ScoredPrompt]: - """Walk a Harbor run directory and yield one ScoredPrompt per episode.""" - for task_dir in sorted(run_root.iterdir()): - if not task_dir.is_dir(): - continue - agent_dir = task_dir / "agent" - if not agent_dir.is_dir(): - continue - task = task_dir.name.split("__", 1)[0] - for episode_dir, idx in _ordered_episode_dirs(agent_dir): - prompt_path = episode_dir / "prompt.txt" - if not prompt_path.is_file(): - continue - try: - blob = prompt_path.read_text() - except OSError: - continue - if not blob.strip(): - continue - yield ScoredPrompt( - run_id=run_id, - source=source, - task=task, - turn_idx=idx, - full=blob, - # Harbor prompts are pre-concatenated blobs (system framing + - # tool history + latest turn), not a structured messages list. - # ``latest`` falls back to the same blob. - latest=blob, - ) - - -def _ordered_episode_dirs(agent_dir: Path) -> Iterator[tuple[Path, int]]: - """Yield (episode_dir, turn_idx) ordered by numeric episode index.""" - ordered: list[tuple[int, Path]] = [] - for child in agent_dir.iterdir(): - if not child.is_dir(): - continue - match = _EPISODE_DIR_PATTERN.match(child.name) - if not match: - continue - ordered.append((int(match.group(1)), child)) - ordered.sort(key=lambda pair: pair[0]) - for idx, path in ordered: - yield path, idx diff --git a/benchmark/dimension_calibration/run.py b/benchmark/dimension_calibration/run.py deleted file mode 100644 index 6bff437e..00000000 --- a/benchmark/dimension_calibration/run.py +++ /dev/null @@ -1,372 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tier 2 calibration runner. - -Walks ``corpus_manifest.yaml``, calls the per-extractor module for each run, -and scores every prompt twice (``full`` and ``latest`` scope) through the -PyO3-backed :class:`switchyard_rust.components.DimensionCollector`. - -Emits under ``report/``: - -* ``data.csv`` — every prompt × scope × dimension score, one row. -* ``tier2_overall.md`` — per-dimension headline numbers. -* ``tier2_by_source.md`` — per-dimension fire rate, sliced by manifest source. -* ``tier2_by_turn.md`` — bucketed turn-index growth slices. - -Run with:: - - uv run python benchmark/dimension_calibration/run.py - -Paths in the manifest resolve relative to the repo root, so adding a new -Harbor run to the corpus is one YAML entry pointing at the run directory. -""" - -from __future__ import annotations - -import asyncio -import csv -import importlib -import statistics -import sys -import time -from collections import Counter, defaultdict -from collections.abc import Iterable, Iterator -from dataclasses import asdict -from pathlib import Path -from typing import Any - -import yaml -from extractors import ScoredPrompt - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.components import ( - DimensionCollector, - get_context_signals, -) -from switchyard_rust.core import ChatRequest - -HERE = Path(__file__).resolve().parent -REPO_ROOT = HERE.parents[1] -MANIFEST_PATH = HERE / "corpus_manifest.yaml" -REPORT_DIR = HERE / "report" - - -def load_manifest() -> list[dict[str, Any]]: - """Load the manifest and resolve paths relative to the repo root.""" - raw = yaml.safe_load(MANIFEST_PATH.read_text()) - runs: list[dict[str, Any]] = [] - for entry in raw.get("runs", []): - run = dict(entry) - run["path"] = str((REPO_ROOT / entry["path"]).resolve()) - runs.append(run) - return runs - - -def stream_prompts(runs: list[dict[str, Any]]) -> Iterator[ScoredPrompt]: - """Walk all runs in the manifest and yield ScoredPrompts in order.""" - for run in runs: - module = importlib.import_module(f"extractors.{run['extractor']}") - path = Path(run["path"]) - if not path.exists(): - print( - f" skip {run['id']}: path does not exist ({path})", - file=sys.stderr, - ) - continue - kwargs: dict[str, Any] = {} - if "source" in run: - kwargs["source"] = run["source"] - yield from module.extract(run["id"], path, **kwargs) - - -def make_request(text: str) -> ChatRequest: - """Wrap a plain text blob as the user message of an OpenAI-chat request. - - The DimensionCollector adapter extracts the user content from - ``messages[]``, so we shape the request body to match what production - traffic looks like at the proxy boundary. - """ - return ChatRequest.openai_chat({ - "model": "calibration", - "messages": [{"role": "user", "content": text}], - }) - - -async def score_one(collector: DimensionCollector, text: str) -> dict[str, Any]: - """Score a single prompt and return the signals as a flat dict.""" - ctx = ProxyContext() - await collector.process(ctx, make_request(text)) - signals = get_context_signals(ctx) - if signals is None: - return {"token_count_estimate": 0, "dims": {}} - return { - "token_count_estimate": signals.token_count_estimate, - "dims": {dim.name: dim.score for dim in signals.dimensions}, - } - - -async def score_all(prompts: Iterable[ScoredPrompt]) -> list[dict[str, Any]]: - """Score every prompt twice (full + latest scopes) and return flat rows.""" - # No-arg constructor picks up the Rust-side `ScoringConfig::default()` — - # the populated keyword set. Passing `ScoringConfig()` from Python would - # build an explicit-empty config and bypass the Rust default. - collector = DimensionCollector() - rows: list[dict[str, Any]] = [] - count = 0 - started_at = time.perf_counter() - for prompt in prompts: - for scope in ("full", "latest"): - text = prompt.full if scope == "full" else prompt.latest - result = await score_one(collector, text) - row = { - **asdict(prompt), - "scope": scope, - "char_len": len(text), - "token_count_estimate": result["token_count_estimate"], - **{f"dim:{name}": score for name, score in result["dims"].items()}, - } - # Trim raw text out of the CSV — keep only metadata. The raw - # prompts live in the source trajectories; replicating them - # here would balloon the CSV and serve no analytical purpose. - row.pop("full", None) - row.pop("latest", None) - rows.append(row) - count += 1 - if count % 500 == 0: - elapsed = time.perf_counter() - started_at - print( - f" scored {count} prompts ({elapsed:.1f}s, " - f"{count / elapsed:.0f}/s)", - file=sys.stderr, - ) - elapsed = time.perf_counter() - started_at - print( - f" scored {count} prompts total in {elapsed:.1f}s " - f"({count / max(elapsed, 1e-9):.0f}/s)", - file=sys.stderr, - ) - return rows - - -# ─── Report writers ────────────────────────────────────────────────────────── - -# Canonical dimension order as the collector emits them, mirrored here so the -# report tables read consistently across runs. -DIMENSIONS = [ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", -] - - -def write_csv(rows: list[dict[str, Any]]) -> Path: - """Dump every (prompt, scope, dimension) row to CSV for offline analysis.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "data.csv" - fieldnames = [ - "run_id", "source", "task", "turn_idx", "scope", "char_len", - "token_count_estimate", - *[f"dim:{d}" for d in DIMENSIONS], - ] - with out.open("w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - writer.writerows(rows) - return out - - -def fire_stats(rows: list[dict[str, Any]], dim: str) -> dict[str, Any]: - """Compute fire-rate + nonzero-score summary for one dimension.""" - key = f"dim:{dim}" - fires: list[float] = [] - total = 0 - for row in rows: - total += 1 - score = row.get(key, 0.0) or 0.0 - if score != 0.0: - fires.append(score) - if total == 0: - return {"n": 0, "fires": 0, "rate": 0.0, - "nonzero_min": None, "nonzero_p50": None, "nonzero_max": None} - return { - "n": total, - "fires": len(fires), - "rate": len(fires) / total, - "nonzero_min": min(fires) if fires else None, - "nonzero_p50": statistics.median(fires) if fires else None, - "nonzero_max": max(fires) if fires else None, - } - - -def write_overall(rows: list[dict[str, Any]]) -> Path: - """Per-dimension headline: fire-rate + nonzero score band, by scope.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_overall.md" - lines: list[str] = [] - lines.append("# Tier-2 calibration — overall per-dimension fire rates") - lines.append("") - lines.append(f"Total prompt observations: **{len(rows)}** " - f"(prompts × 2 scopes).") - lines.append("") - for scope in ("full", "latest"): - scoped = [r for r in rows if r["scope"] == scope] - lines.append(f"## scope = `{scope}` ({len(scoped)} rows)") - lines.append("") - lines.append("| Dimension | Fire rate | Fires | Nonzero min / p50 / max |") - lines.append("|---|---:|---:|---|") - for dim in DIMENSIONS: - s = fire_stats(scoped, dim) - band = "—" if s["fires"] == 0 else \ - f"{s['nonzero_min']:.2f} / {s['nonzero_p50']:.2f} / {s['nonzero_max']:.2f}" - lines.append( - f"| `{dim}` | {s['rate']:.1%} | {s['fires']} | {band} |" - ) - lines.append("") - # Aggregate scalars on the same scope - tc = [r["token_count_estimate"] for r in scoped] - if tc: - lines.append( - f"`token_count_estimate`: min={min(tc)}, " - f"p50={statistics.median(tc):.0f}, max={max(tc)}, " - f"mean={statistics.fmean(tc):.0f}" - ) - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_by_source(rows: list[dict[str, Any]]) -> Path: - """Per-dimension fire-rate sliced by trajectory source (hermes vs harbor).""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_by_source.md" - sources = sorted({r["source"] for r in rows}) - lines: list[str] = [] - lines.append("# Tier-2 calibration — fire rate by source") - lines.append("") - lines.append("Restricted to `scope=full` (production-realistic blob).") - lines.append("") - header = "| Dimension | " + " | ".join(sources) + " |" - sep = "|---|" + "|".join(["---:"] * len(sources)) + "|" - lines.append(header) - lines.append(sep) - for dim in DIMENSIONS: - per_source: list[str] = [] - for src in sources: - scoped = [r for r in rows if r["source"] == src and r["scope"] == "full"] - s = fire_stats(scoped, dim) - per_source.append(f"{s['rate']:.1%}") - lines.append(f"| `{dim}` | " + " | ".join(per_source) + " |") - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_by_turn(rows: list[dict[str, Any]]) -> Path: - """Fire-rate sliced by turn-position bucket — diagnoses trajectory growth.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_by_turn.md" - # Three buckets: turn 0 (task framing), early (1-4), mid+ (5+). - def bucket(idx: int) -> str: - if idx == 0: - return "turn=0" - if idx <= 4: - return "1-4" - return "5+" - - lines: list[str] = [] - lines.append("# Tier-2 calibration — fire rate by turn position") - lines.append("") - lines.append("Restricted to `scope=full`. Buckets: `turn=0` (task framing) / " - "`1-4` (early loop) / `5+` (deep loop).") - lines.append("") - buckets = ["turn=0", "1-4", "5+"] - header = "| Dimension | " + " | ".join(buckets) + " |" - lines.append(header) - lines.append("|---|" + "|".join(["---:"] * len(buckets)) + "|") - for dim in DIMENSIONS: - per_bucket: list[str] = [] - for b in buckets: - scoped = [ - r for r in rows - if r["scope"] == "full" and bucket(r["turn_idx"]) == b - ] - s = fire_stats(scoped, dim) - per_bucket.append(f"{s['rate']:.1%}") - lines.append(f"| `{dim}` | " + " | ".join(per_bucket) + " |") - lines.append("") - # Per-bucket row counts for context - counts: Counter[str] = Counter( - bucket(r["turn_idx"]) for r in rows if r["scope"] == "full" - ) - lines.append( - "Row counts per bucket (scope=full): " - + ", ".join(f"{b}={counts.get(b, 0)}" for b in buckets) - ) - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_per_run_breakdown(rows: list[dict[str, Any]]) -> Path: - """Per-run row counts so reviewers see corpus composition.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_corpus_composition.md" - by_run: defaultdict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - if row["scope"] == "full": - by_run[row["run_id"]].append(row) - lines: list[str] = [] - lines.append("# Tier-2 calibration — corpus composition") - lines.append("") - lines.append("| Run | Prompts (scope=full) | Tasks | Avg turn idx |") - lines.append("|---|---:|---:|---:|") - for run_id in sorted(by_run): - scoped = by_run[run_id] - tasks = len({r["task"] for r in scoped}) - avg_turn = statistics.fmean(r["turn_idx"] for r in scoped) - lines.append(f"| `{run_id}` | {len(scoped)} | {tasks} | {avg_turn:.1f} |") - lines.append("") - out.write_text("\n".join(lines)) - return out - - -async def main() -> int: - print("loading manifest…", file=sys.stderr) - runs = load_manifest() - print(f" {len(runs)} runs declared", file=sys.stderr) - - print("scoring prompts (this is one DimensionCollector pass per scope)…", - file=sys.stderr) - rows = await score_all(stream_prompts(runs)) - - print("writing reports…", file=sys.stderr) - csv_path = write_csv(rows) - overall_path = write_overall(rows) - by_source_path = write_by_source(rows) - by_turn_path = write_by_turn(rows) - composition_path = write_per_run_breakdown(rows) - - print("done.", file=sys.stderr) - print() - print(f" csv: {csv_path.relative_to(REPO_ROOT)}") - print(f" overall report: {overall_path.relative_to(REPO_ROOT)}") - print(f" by-source report: {by_source_path.relative_to(REPO_ROOT)}") - print(f" by-turn report: {by_turn_path.relative_to(REPO_ROOT)}") - print(f" corpus composition: {composition_path.relative_to(REPO_ROOT)}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(asyncio.run(main())) diff --git a/benchmark/score_staged_run.py b/benchmark/score_staged_run.py new file mode 100644 index 00000000..7bfafba7 --- /dev/null +++ b/benchmark/score_staged_run.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Score a benchmark run directory via the stage-router Rust scorer. + +Reads trajectory.json from each completed task, feeds each tool-use turn through: + DimensionCollector.process(ctx, request) → pick_capable_first / pick_efficient_first + +The picker functions replicate live routing exactly: overrides, scorer, and fall_open. + +Usage: + uv run python benchmark/score_run.py --run benchmark/tb_runs/ + uv run python benchmark/score_run.py --run benchmark/tb_runs/ \\ + --output /tmp/scores.jsonl --threshold 0.20 --window 3 +""" +import argparse +import asyncio +import csv +import json +import sys +from collections import defaultdict +from pathlib import Path +from statistics import mean + +from switchyard.lib.processors.stage_router.picker import ( + CAPABLE, + pick_capable_first, + pick_efficient_first, +) +from switchyard_rust.components import ( + DimensionCollector, + get_tool_result_signal, + stage_score_signal, +) +from switchyard_rust.core import ChatRequest, ProxyContext + +RECENT_WINDOW = 3 + + +def _tool_use_id(message: str) -> str: + parts = message.split() + return parts[-1] if len(parts) >= 2 else f"tu_{id(message)}" + + +async def score_trajectory( + traj: dict, + reward: float | None, + task_name: str, + trial_name: str, + run_id: str, + recent_window: int, + confidence_threshold: float, +) -> list[dict]: + steps = traj.get("steps", []) + + # One DimensionCollector per task — accumulates state across turns + dc = DimensionCollector(recent_window=recent_window) + await dc.startup() + + rows: list[dict] = [] + messages: list[dict] = [] + + for s in steps: + if s["source"] == "user" and not (s.get("extra") or {}).get("is_sidechain"): + messages.append({"role": "user", "content": [{"type": "text", "text": s["message"]}]}) + break + + if not messages: + return rows + + for s in steps: + if s["source"] != "agent": + continue + extra = s.get("extra") or {} + if extra.get("is_sidechain"): + continue + + tool_name = extra.get("tool_use_name") + if not tool_name: + text = s.get("message", "") + if text: + messages.append({"role": "assistant", "content": [{"type": "text", "text": text}]}) + continue + + tool_use_id = _tool_use_id(s.get("message", "")) + raw_args = extra.get("raw_arguments") or {} + is_error = bool(extra.get("tool_result_is_error", False)) + + metadata = extra.get("metadata") or {} + raw_result = metadata.get("raw_tool_result") or {} + content = raw_result.get("content", "") + if not isinstance(content, str): + content = json.dumps(content) + + messages.append({ + "role": "assistant", + "content": [{"type": "tool_use", "id": tool_use_id, "name": tool_name, "input": raw_args}], + }) + messages.append({ + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": content, "is_error": is_error}], + }) + + # Build Anthropic ChatRequest and process once through DimensionCollector + ctx = ProxyContext() + request = ChatRequest.anthropic({ + "model": "claude-opus-4-8", + "max_tokens": 8096, + "messages": messages, + }) + await dc.process(ctx, request) + + signal = get_tool_result_signal(ctx) + if signal is None: + continue + + # Raw score for analysis (Rust scorer): wrong signals score positive + # (→CAPABLE), progress negative (→EFFICIENT). Picker-independent, so the + # histogram shows the capable/efficient separation directly. + score, confidence = stage_score_signal(signal) + + # Actual picker decisions on the same ctx — both just read the signal, + # no extra dc.process() calls needed + tier_cf = await pick_capable_first(ctx, confidence_threshold) + tier_ef = await pick_efficient_first(ctx, confidence_threshold) + + rows.append({ + "task_name": task_name, + "trial_name": trial_name, + "run_id": run_id, + "reward": reward, + "turn_depth": signal.turn_depth, + "score": score, + "confidence": confidence, + "tool_name": tool_name, + "is_error": is_error, + "write_count": signal.write_count, + "edit_count": signal.edit_count, + "read_count": signal.read_count, + "no_error_streak": signal.no_error_streak, + "pure_bash_streak": signal.pure_bash_streak, + "tests_passed": signal.tests_passed, + "pick_cf": tier_cf, # CAPABLE=1, EFFICIENT=0 + "pick_ef": tier_ef, + }) + + return rows + + +def band(score: float, threshold: float) -> str: + if score >= threshold: + return "strong_clear" + if score > 0: + return "strong_uncertain" + if score <= -threshold: + return "weak_clear" + if score < 0: + return "weak_uncertain" + return "zero" + + +def write_per_task_csv(all_rows: list[dict], csv_path: Path, threshold: float) -> None: + groups: dict[tuple, list] = defaultdict(list) + for r in all_rows: + groups[(r["run_id"], r["task_name"])].append(r) + + fieldnames = [ + "run_id", "task_name", "reward", "n_turns", + "mean_score", "mean_confidence", + "pct_strong_clear", "pct_strong_uncertain", "pct_weak_uncertain", "pct_weak_clear", + "opus_pct_cf", "nemotron_pct_cf", "opus_pct_ef", "nemotron_pct_ef", + ] + rows_out = [] + for (run_id, task_name), turns in sorted(groups.items()): + n = len(turns) + scores = [t["score"] for t in turns] + confs = [t["confidence"] for t in turns] + reward = turns[0]["reward"] + bands = [band(s, threshold) for s in scores] + n_sc = bands.count("strong_clear") + n_su = bands.count("strong_uncertain") + n_wu = bands.count("weak_uncertain") + n_wc = bands.count("weak_clear") + n_cf_opus = sum(1 for t in turns if t["pick_cf"] == CAPABLE) + n_ef_opus = sum(1 for t in turns if t["pick_ef"] == CAPABLE) + rows_out.append({ + "run_id": run_id, + "task_name": task_name, + "reward": reward, + "n_turns": n, + "mean_score": round(mean(scores), 4), + "mean_confidence": round(mean(confs), 4), + "pct_strong_clear": round(n_sc / n, 4), + "pct_strong_uncertain": round(n_su / n, 4), + "pct_weak_uncertain": round(n_wu / n, 4), + "pct_weak_clear": round(n_wc / n, 4), + "opus_pct_cf": round(n_cf_opus / n, 4), + "nemotron_pct_cf": round(1 - n_cf_opus / n, 4), + "opus_pct_ef": round(n_ef_opus / n, 4), + "nemotron_pct_ef": round(1 - n_ef_opus / n, 4), + }) + + csv_path.parent.mkdir(parents=True, exist_ok=True) + with open(csv_path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames) + w.writeheader() + w.writerows(rows_out) + print(f"Per-task CSV → {csv_path} ({len(rows_out)} rows)") + + +async def score_run(run_dir: Path, recent_window: int, threshold: float) -> list[dict]: + jobs_dirs = list(run_dir.glob("jobs/*/")) or [run_dir] + run_id = run_dir.name + + all_rows: list[dict] = [] + for jobs_dir in jobs_dirs: + task_dirs = [d for d in jobs_dir.iterdir() if d.is_dir() and d.name != "verifier"] + print(f"{jobs_dir.name}: {len(task_dirs)} task dirs") + + for task_dir in sorted(task_dirs): + traj_path = task_dir / "agent" / "trajectory.json" + result_path = task_dir / "result.json" + if not traj_path.exists(): + continue + + traj = json.loads(traj_path.read_text()) + reward: float | None = None + task_name = task_dir.name + if result_path.exists(): + res = json.loads(result_path.read_text()) + task_name = res.get("task_name") or task_name + r = (res.get("verifier_result") or {}).get("rewards", {}).get("reward") + reward = float(r) if r is not None else None + + rows = await score_trajectory( + traj, reward, task_name, task_dir.name, run_id, recent_window, threshold + ) + all_rows.extend(rows) + print(f" {task_dir.name}: {len(rows)} turns, reward={reward}") + + return all_rows + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True, help="Path to run directory under benchmark/tb_runs/") + parser.add_argument("--output", help="Per-turn JSONL (default: /tmp/-scores.jsonl)") + parser.add_argument("--csv", help="Per-task CSV (default: /tmp/-per-task.csv)") + parser.add_argument("--threshold", type=float, default=0.20) + parser.add_argument("--window", type=int, default=RECENT_WINDOW) + args = parser.parse_args() + + run_dir = Path(args.run) + if not run_dir.exists(): + sys.exit(f"Run directory not found: {run_dir}") + + jsonl_out = Path(args.output or f"/tmp/{run_dir.name}-scores.jsonl") + csv_out = Path(args.csv or f"/tmp/{run_dir.name}-per-task.csv") + + all_rows = await score_run(run_dir, args.window, args.threshold) + + jsonl_out.parent.mkdir(parents=True, exist_ok=True) + with open(jsonl_out, "w") as fh: + for row in all_rows: + fh.write(json.dumps(row) + "\n") + print(f"Per-turn JSONL → {jsonl_out} ({len(all_rows)} rows)") + + if all_rows: + write_per_task_csv(all_rows, csv_out, args.threshold) + total = len(all_rows) + scores = [r["score"] for r in all_rows] + cf_opus = sum(1 for r in all_rows if r["pick_cf"] == CAPABLE) + ef_opus = sum(1 for r in all_rows if r["pick_ef"] == CAPABLE) + print(f"\nGlobal summary ({total} turns, threshold={args.threshold}):") + for bn in ["strong_clear", "strong_uncertain", "weak_uncertain", "weak_clear"]: + n = sum(1 for s in scores if band(s, args.threshold) == bn) + print(f" {bn:<22} {n:>5} ({100*n/total:.1f}%)") + print(f" cf → Opus: {cf_opus}/{total} ({100*cf_opus/total:.1f}%)") + print(f" ef → Opus: {ef_opus}/{total} ({100*ef_opus/total:.1f}%)") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/crates/libsy/src/signal/tool_signals.rs b/crates/libsy/src/signal/tool_signals.rs index a7020fa2..91b4ffaa 100644 --- a/crates/libsy/src/signal/tool_signals.rs +++ b/crates/libsy/src/signal/tool_signals.rs @@ -65,7 +65,14 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ ( "no_such_file", HARD, - &["filenotfounderror:", "no such file or directory"], + &[ + "filenotfounderror:", + "no such file or directory", + // Claude Code Read-tool miss. Anchored as "file does not exist" (not a + // bare "does not exist", which fires on `ls` output and prose) — trace- + // mined across 1006 local trajectories at 22 true / 2 false positives. + "file does not exist", + ], ), // SOFT: plain non-zero exit without a recognisable exception traceback. ( @@ -132,8 +139,7 @@ static BASH_READ_PATTERNS: &[&str] = &[ static READ_TOOL_NAMES: &[&str] = &["read", "view"]; -// Planning / scratchpad tool calls. Used by Opus as a struggle indicator -// in the capable-first picker direction. +// Planning / scratchpad tool calls — investigative (non-producing) activity. // `update_plan` is codex's equivalent of `todowrite`. static PLAN_TOOL_NAMES: &[&str] = &["todowrite", "todo_write", "todo", "update_plan"]; @@ -168,13 +174,13 @@ static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "er // "0 errors" summaries on a clean run are not misread as failures. static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"]; -/// Default sliding-window size for `recent_*` counts. +/// Default sliding-window size for `recent_*` counts and windowed severity. /// -/// Calibrated against agent trajectories where a 3-call horizon is enough -/// to capture the "what is the agent doing right now" signal without -/// over-smoothing short tasks. Override per-extractor by calling -/// [`extract_tool_signals_with_window`] directly. -pub const DEFAULT_RECENT_WINDOW: usize = 3; +/// A 5-call horizon captures "what is the agent doing right now" while keeping +/// signals sticky — an error or stall persists a few recovery turns instead of +/// flickering off the moment one clean result lands. Override per-extractor by +/// calling [`extract_tool_signals_with_window`] directly. +pub const DEFAULT_RECENT_WINDOW: usize = 5; // ─── output type ───────────────────────────────────────────────────────────── @@ -182,18 +188,19 @@ pub const DEFAULT_RECENT_WINDOW: usize = 3; /// via [`crate::get_tool_result_signal`]. #[derive(Clone, Debug, Default)] pub struct ToolSignals { + /// Max severity across the recent window (last `recent_window` tool results): /// `0.0` clean · `0.3` soft (exit_nonzero) · `0.7` hard · `1.0` critical. + /// Windowed so an error persists through the recovery turns instead of clearing + /// the instant the next result is clean. pub severity: f32, - /// Error pattern names that fired in the most recent tool result. - pub patterns: Vec, /// Consecutive clean tool results back from the most recent. `0` if the last failed. pub no_error_streak: u32, pub edit_count: u32, pub write_count: u32, /// Read-type calls (Read tool + read-like Bash). Used by the build-pit gate. pub read_count: u32, - /// TodoWrite tool calls. Strong fail predictor for Opus; used by the - /// `capable_first` drop-to-weak gate. + /// TodoWrite / planning tool calls. Investigative (non-producing) activity — + /// recent todowrites distinguish `exploring` from `spinning` in the scorer. pub todowrite_count: u32, /// Edit-type calls within the last [`RECENT_WINDOW`] tool calls. pub recent_edit_count: u32, @@ -203,15 +210,21 @@ pub struct ToolSignals { pub recent_read_count: u32, /// TodoWrite calls within the last [`RECENT_WINDOW`] tool calls. pub recent_todowrite_count: u32, - /// Consecutive trailing tool calls that hit no Write/Edit/Read/Plan - /// patterns — proxy for the "stuck in non-Read Bash" build-pit loop. + /// Consecutive trailing tool calls in the `Other` category (no Write/Edit/Read/ + /// Plan match). Surfaced in the classifier state summary; not scored directly. pub pure_bash_streak: u32, /// At least one of the last three tool results matched a test-pass pattern. pub tests_passed: bool, - /// Message-count proxy for turn depth. + /// Message-count proxy for turn depth. Wire-format dependent (Anthropic batches + /// tool results into fewer messages than OpenAI-chat), so gates keyed on it are + /// approximate across request origins. pub turn_depth: u32, - /// Char count of the last user message. - pub prompt_char_count: u32, + /// The request carries a context-compaction summary (the agent's context was + /// summarised after overflowing). Compaction resets the router's accumulated + /// signals, so a task that was on the strong tier de-escalates back to weak — the + /// picker uses this to force + hold the strong tier. Self-latching: the summary + /// stays in the context prefix on every subsequent turn. + pub compacted: bool, } impl ToolSignals { @@ -269,7 +282,7 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { // ─── extraction entry point ─────────────────────────────────────────────────── -/// Extract all tool-execution signals from a [`ChatRequest`]. +/// Extract all tool-execution signals from a [`Request`]. /// /// Dispatches on the request's wire format: /// @@ -280,7 +293,7 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { /// * **OpenAI responses** — `input[]` with `type: "function_call_output"`; /// `type: "function_call"` for call names. /// -/// Returns [`ToolSignals::default()`] when the body is absent or +/// Returns [`ToolSignals::default()`] when the wire format or body is absent or /// the messages list is empty — callers can always read `signal.severity`. fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals { let Some(Some(wire_format)) = request.metadata.as_ref().map(|m| m.wire_format) else { @@ -293,14 +306,17 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> return ToolSignals::default(); }; - match wire_format { + let (mut signal, entries) = match wire_format { WireFormat::OpenAiChat => { let messages = obj .get("messages") .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_openai_chat(messages, recent_window) + ( + extract_from_messages_openai_chat(messages, recent_window), + messages, + ) } WireFormat::AnthropicMessages => { let messages = obj @@ -308,7 +324,10 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_anthropic(messages, recent_window) + ( + extract_from_messages_anthropic(messages, recent_window), + messages, + ) } WireFormat::OpenAiResponses => { let items = obj @@ -316,17 +335,43 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_input_responses(items, recent_window) + (extract_from_input_responses(items, recent_window), items) } - } + }; + + // Compaction is detected anywhere in the message/item contents (the summary stays + // in the prefix on every subsequent turn, so this self-latches once it fires). + signal.compacted = entries.iter().any(|m| { + m.as_object() + .is_some_and(|o| content_has_compaction_marker(o.get("content"))) + }); + signal } // ─── format-specific extractors ────────────────────────────────────────────── +/// Distinctive preamble Claude Code injects as a user message when it compacts an +/// overflowed context. Matched case-insensitively; normal task text never contains it. +const COMPACTION_MARKER: &str = "session is being continued"; + +/// True when a user-message content (string or text blocks) carries the compaction +/// summary preamble. +fn content_has_compaction_marker(content: Option<&Value>) -> bool { + match content { + Some(Value::String(s)) => s.to_lowercase().contains(COMPACTION_MARKER), + Some(Value::Array(blocks)) => blocks.iter().any(|b| { + b.as_object() + .and_then(|o| o.get("text")) + .and_then(Value::as_str) + .is_some_and(|t| t.to_lowercase().contains(COMPACTION_MARKER)) + }), + _ => false, + } +} + fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -367,26 +412,16 @@ fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) - } } } - "user" => { - prompt_char_count = user_message_char_count(obj.get("content")); - } _ => {} } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -396,31 +431,14 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> match role { "user" => { if let Some(Value::Array(blocks)) = content { - let mut saw_text = false; for block in blocks { let Some(b) = block.as_object() else { continue }; - match b.get("type").and_then(Value::as_str) { - Some("tool_result") => { - if let Some(text) = content_to_text(b.get("content")) { - tool_texts.push(text); - } + if b.get("type").and_then(Value::as_str) == Some("tool_result") { + if let Some(text) = content_to_text(b.get("content")) { + tool_texts.push(text); } - Some("text") => { - if let Some(t) = b.get("text").and_then(Value::as_str) { - prompt_char_count = t.len() as u32; - saw_text = true; - } - } - _ => {} - } - } - if !saw_text { - if let Some(text_val) = content { - prompt_char_count = user_message_char_count(Some(text_val)); } } - } else { - prompt_char_count = user_message_char_count(content); } } "assistant" => { @@ -450,26 +468,18 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for item in items { let Some(obj) = item.as_object() else { continue; }; let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); match item_type { "function_call_output" => { @@ -495,21 +505,11 @@ fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolSi command, }); } - _ => { - if role == "user" { - prompt_char_count = user_message_char_count(obj.get("content")); - } - } + _ => {} } } - build_signal( - tool_texts, - tool_calls, - items.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, items.len() as u32, recent_window) } // ─── aggregation ───────────────────────────────────────────────────────────── @@ -518,14 +518,21 @@ fn build_signal( tool_texts: Vec, tool_calls: Vec, turn_depth: u32, - prompt_char_count: u32, recent_window: usize, ) -> ToolSignals { - let (severity, patterns) = if let Some(last) = tool_texts.last() { - classify_text(last) - } else { - (0.0, Vec::new()) - }; + // Windowed severity: take the MAX severity across the last `recent_window` tool + // results rather than only the last one. An error's severity then persists for + // the recent window and decays out of it — parallel to the windowed `recent_*` + // counts — so a fix written a couple of turns after an error still routes on the + // error signal instead of the router flapping straight back to the weak tier. + let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); + let mut severity = 0.0f32; + for text in &tool_texts[sev_start..] { + let (sev, _patterns) = classify_text(text); + if sev > severity { + severity = sev; + } + } let no_error_streak = compute_no_error_streak(&tool_texts); @@ -581,11 +588,10 @@ fn build_signal( } } - let tests_passed = detect_tests_passed(&tool_texts); + let tests_passed = detect_tests_passed(&tool_texts, recent_window); ToolSignals { severity, - patterns, no_error_streak, edit_count, write_count, @@ -598,7 +604,9 @@ fn build_signal( pure_bash_streak, tests_passed, turn_depth, - prompt_char_count, + // Set by extract_tool_signals_with_window after the format-specific extract, + // which scans all message contents for the compaction marker. + compacted: false, } } @@ -628,24 +636,6 @@ fn content_to_text(content: Option<&Value>) -> Option { } } -/// Character count of a user-message content value. -fn user_message_char_count(content: Option<&Value>) -> u32 { - match content { - Some(Value::String(s)) => s.len() as u32, - Some(Value::Array(blocks)) => blocks - .iter() - .filter_map(|b| { - b.as_object() - .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - }) - .map(|s| s.len() as u32) - .sum(), - _ => 0, - } -} - /// Match `text` against the error pattern table. /// /// Returns `(max_severity, matched_pattern_names)`. @@ -674,13 +664,9 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 { streak } -fn detect_tests_passed(tool_texts: &[String]) -> bool { - let recent = if tool_texts.len() > 3 { - &tool_texts[tool_texts.len() - 3..] - } else { - tool_texts - }; - recent.iter().any(|text| { +fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool { + let start = tool_texts.len().saturating_sub(recent_window.max(1)); + tool_texts[start..].iter().any(|text| { let lower = text.to_lowercase(); TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) @@ -782,6 +768,23 @@ mod tests { assert_eq!(sev, HARD); } + #[test] + fn file_does_not_exist_is_hard() { + // Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives). + let (sev, patterns) = + classify_text("Error: File does not exist. Note: current working directory is /app."); + assert_eq!(sev, HARD); + assert!(patterns.contains(&"no_such_file".to_string())); + } + + #[test] + fn bare_does_not_exist_stays_clean() { + // Precision guard: only the anchored "file does not exist" fires, so a bare + // "does not exist" in prose or directory output must not trip a false error. + let (sev, _) = classify_text("The directory does not exist yet, creating it now."); + assert_eq!(sev, 0.0); + } + #[test] fn no_error_streak_all_clean() { let texts = vec!["ok".to_string(), "all good".to_string()]; @@ -800,38 +803,48 @@ mod tests { #[test] fn tests_passed_detects_pytest_output() { - assert!(detect_tests_passed(&[ - "====== 5 passed in 0.12s ======".to_string() - ])); + assert!(detect_tests_passed( + &["====== 5 passed in 0.12s ======".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_ignores_partial_failures() { - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ])); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); + } + + #[test] + fn severity_is_windowed_over_recent_results() { + // An error two results back, then two clean results. + let request = openai_chat_request(json!({ + "messages": [ + {"role": "tool", "tool_call_id": "1", + "content": "Traceback (most recent call last):\n ValueError"}, + {"role": "tool", "tool_call_id": "2", "content": "ok"}, + {"role": "tool", "tool_call_id": "3", "content": "ok"}, + ] + })); + // window covers the error → severity persists (max over the window) + assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD); + // window of 1 sees only the last (clean) result → severity has decayed out + assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0); } #[test] fn extract_openai_chat_tool_results() { - let raw_request = Some(json!({ + let request = openai_chat_request(json!({ "messages": [ {"role": "user", "content": "do something"}, {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, {"role": "tool", "tool_call_id": "1", "content": "Traceback (most recent call last):\n ValueError"}, ] })); - let request = Request { - raw_request, - metadata: Some(Metadata { - wire_format: Some(WireFormat::OpenAiChat), - ..Default::default() - }), - ..Default::default() - }; let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, HARD); - assert!(sig.patterns.contains(&"traceback".to_string())); assert_eq!(sig.edit_count, 1); assert_eq!(sig.turn_depth, 3); } @@ -865,9 +878,9 @@ mod tests { } #[test] - fn recent_window_counts_only_last_three_tool_calls() { - // 5 writes + 1 edit at the end → recent window of 3 should see - // 1 edit + 2 writes (not all 5 writes). + fn recent_window_counts_only_last_default_window_tool_calls() { + // 5 writes + 1 edit at the end → the default window (5) should see + // the last 5 calls: 1 edit + 4 writes (not all 6 calls). let request = openai_chat_request(json!({ "messages": [ {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, @@ -887,7 +900,7 @@ mod tests { let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 5); assert_eq!(sig.edit_count, 1); - assert_eq!(sig.recent_write_count, 2); + assert_eq!(sig.recent_write_count, 4); assert_eq!(sig.recent_edit_count, 1); } @@ -921,6 +934,29 @@ mod tests { assert_eq!(wide.recent_edit_count, 1); } + #[test] + fn compaction_marker_sets_compacted() { + // The compaction summary is a user message carrying Claude Code's preamble. + let request = openai_chat_request(json!({ + "messages": [ + {"role": "user", "content": "This session is being continued from a previous conversation that ran out of context."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(ToolSignals::from_request(&request, None).compacted); + } + + #[test] + fn no_compaction_marker_stays_uncompacted() { + let request = openai_chat_request(json!({ + "messages": [ + {"role": "user", "content": "Write a script that parses the log file."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(!ToolSignals::from_request(&request, None).compacted); + } + #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. @@ -983,47 +1019,55 @@ mod tests { #[test] fn tests_passed_detects_pytest_with_failure_block() { // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ])); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_cargo_clean_summary() { // Cargo's clean-run summary contains "0 failed" — must not trip the // failure list (regression: previously substring-matched "failed"). - assert!(detect_tests_passed(&[ - "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string() - ])); + assert!(detect_tests_passed( + &["running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_rejects_cargo_real_failure() { // Cargo's actual-failure summary: nonzero count before "failed". - assert!(!detect_tests_passed(&[ - "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string() - ])); + assert!(!detect_tests_passed( + &["running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_go_clean_summary() { // Go test's clean-run "0 errors" must not trip (regression). - assert!(detect_tests_passed(&[ - "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string() - ])); + assert!(detect_tests_passed( + &["ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_pytest_zero_errors() { // Pytest long-form: "0 errors in 0.3s" on a clean run. - assert!(detect_tests_passed(&[ - "5 passed, 0 errors in 0.30s".to_string() - ])); + assert!(detect_tests_passed( + &["5 passed, 0 errors in 0.30s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_detects_diy_checkmark() { - assert!(detect_tests_passed(&["✓ all checks passed".to_string()])); + assert!(detect_tests_passed( + &["✓ all checks passed".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] diff --git a/crates/switchyard-components/src/dimension_collector/config.rs b/crates/switchyard-components/src/dimension_collector/config.rs deleted file mode 100644 index d0c93cdc..00000000 --- a/crates/switchyard-components/src/dimension_collector/config.rs +++ /dev/null @@ -1,351 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Config types and scored-signal records for the dimension collector. -//! -//! Mirrors `ClawRouter/src/router/types.ts` (MIT). Only the fields the -//! collector itself consumes are reproduced here; tier maps, promotions, -//! and model pricing live with the eventual rules estimator, not the -//! context-signal layer. - -/// One scorer's output: dimension name, signed score, optional human-readable signal. -/// -/// Scores roughly in `[-1, 1]`. Individual scorers narrow that range -/// (`simple_indicators` is non-positive, `code_presence` is non-negative). -#[derive(Clone, Debug, PartialEq)] -pub struct DimensionScore { - pub name: &'static str, - pub score: f32, - pub signal: Option, -} - -impl DimensionScore { - /// Convenience constructor for the no-signal case (score 0.0). - pub fn zero(name: &'static str) -> Self { - Self { - name, - score: 0.0, - signal: None, - } - } -} - -/// Token-count thresholds for `score_token_count`. -/// -/// Below `short` → score `-1.0` (looks SIMPLE). Above `long` → score -/// `+1.0` (looks COMPLEX). In-between → `0.0`. -#[derive(Clone, Debug, PartialEq)] -pub struct TokenCountThresholds { - pub short: u32, - pub long: u32, -} - -impl Default for TokenCountThresholds { - fn default() -> Self { - Self { - short: 50, - long: 500, - } - } -} - -/// Pre-lowercased keyword list for substring-match scorers. -/// -/// Keyword lists are lowercased once at config construction so the -/// per-request hot path does only the prompt-side `to_lowercase()` plus -/// `O(len(keywords) × len(text))` substring scans. -#[derive(Clone, Debug, Default, PartialEq)] -pub struct Keywords(Vec); - -impl Keywords { - /// Constructs a lowercased keyword set from any iterator of strings. - pub fn new(items: impl IntoIterator>) -> Self { - let lowered: Vec = items - .into_iter() - .map(|item| item.as_ref().to_lowercase()) - .collect(); - Self(lowered) - } - - /// Constructs a keyword set from a static list of already-lowercase - /// `&'static str` entries — used by [`ScoringConfig::clawrouter_defaults`] - /// so the per-construction lowercase pass is skipped. - pub fn from_static(items: &[&'static str]) -> Self { - Self(items.iter().map(|s| (*s).to_string()).collect()) - } - - /// Returns the underlying lowercased keyword slice. - pub fn as_slice(&self) -> &[String] { - &self.0 - } - - /// Returns whether the keyword set is empty. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -/// Per-scorer configuration consumed by the dimension collector. -/// -/// Only the subset of `ScoringConfig` the collector needs lives here. -/// Tier boundaries, dimension weights, confidence calibration, and -/// model-pricing knobs belong to a separate rules estimator — putting -/// them here would couple the context-signal layer to scoring policy. -#[derive(Clone, Debug, PartialEq)] -pub struct ScoringConfig { - pub token_count: TokenCountThresholds, - pub code_keywords: Keywords, - pub reasoning_keywords: Keywords, - pub simple_keywords: Keywords, - pub technical_keywords: Keywords, - pub creative_keywords: Keywords, - pub imperative_verbs: Keywords, - pub constraint_indicators: Keywords, - pub output_format_keywords: Keywords, - pub reference_keywords: Keywords, - pub negation_keywords: Keywords, - pub domain_specific_keywords: Keywords, -} - -impl Default for ScoringConfig { - fn default() -> Self { - Self::clawrouter_defaults() - } -} - -impl ScoringConfig { - /// Returns the ClawRouter-aligned default keyword set. - /// - /// Trailing-space suffixes (e.g. `"def "`, `"not "`) prevent false - /// substring matches inside common longer words (`"definition"`, - /// `"another"`). The collector lower-cases the prompt once per request - /// and matches each keyword as a substring, so suffix discipline is - /// the cheapest way to keep precision up. - pub fn clawrouter_defaults() -> Self { - Self { - token_count: TokenCountThresholds::default(), - code_keywords: Keywords::from_static(defaults::CODE), - reasoning_keywords: Keywords::from_static(defaults::REASONING), - simple_keywords: Keywords::from_static(defaults::SIMPLE), - technical_keywords: Keywords::from_static(defaults::TECHNICAL), - creative_keywords: Keywords::from_static(defaults::CREATIVE), - imperative_verbs: Keywords::from_static(defaults::IMPERATIVE_VERBS), - constraint_indicators: Keywords::from_static(defaults::CONSTRAINT_INDICATORS), - output_format_keywords: Keywords::from_static(defaults::OUTPUT_FORMAT), - reference_keywords: Keywords::from_static(defaults::REFERENCE), - negation_keywords: Keywords::from_static(defaults::NEGATION), - domain_specific_keywords: Keywords::from_static(defaults::DOMAIN_SPECIFIC), - } - } -} - -/// Static keyword tables for [`ScoringConfig::clawrouter_defaults`]. -/// All entries lowercase by construction; retuning here is a pure data diff. -mod defaults { - pub static CODE: &[&str] = &[ - "def ", - "class ", - "function", - "async ", - "await ", - "import ", - "from ", - "return ", - "fn ", - "struct ", - "impl ", - "trait ", - "pub ", - "mod ", - "package ", - "interface ", - "namespace ", - "let ", - "const ", - "```", - "->", - "=>", - "{}", - "[]", - ]; - - pub static REASONING: &[&str] = &[ - "prove", - "theorem", - "derive", - "step by step", - "let's think", - "reasoning", - "deduce", - "induction", - "lemma", - "proof", - "show that", - "demonstrate", - "given that", - "therefore", - "hence ", - "thus ", - "implies", - "because of", - ]; - - // "no " and "ok " removed — collide with "no longer", "no need", "ok with". - pub static SIMPLE: &[&str] = &[ - "hello", - "hi ", - "thanks", - "thank you", - "what is", - "what's", - "define ", - "meaning of", - "summary of", - "translate ", - ]; - - pub static TECHNICAL: &[&str] = &[ - "distributed", - "concurrent", - "kubernetes", - "container", - "algorithm", - "protocol", - "throughput", - "latency", - "bandwidth", - "consistency", - "compiler", - "interpreter", - "garbage collection", - "memory leak", - "race condition", - "deadlock", - "cache", - "transaction", - "atomic", - "encryption", - "compression", - "deserialize", - "serialize", - ]; - - pub static CREATIVE: &[&str] = &[ - "story", - "poem", - "novel", - "lyrics", - "creative", - "brainstorm", - "imagine", - "fictional", - "narrative", - "character", - "plot", - "metaphor", - "analogy", - "essay", - ]; - - // Over-common verbs ("add", "update", "change", "make", "write") - // removed — they appear in nearly every request and gave the dimension - // no discriminating power. - pub static IMPERATIVE_VERBS: &[&str] = &[ - "implement ", - "design ", - "refactor ", - "architect ", - "engineer ", - "compose ", - "scaffold ", - "bootstrap ", - ]; - - pub static CONSTRAINT_INDICATORS: &[&str] = &[ - "must ", - "should ", - "at most", - "at least", - "within", - "no more than", - "fewer than", - "exactly", - "o(n)", - "o(1)", - "o(log", - "linear time", - "constant time", - "bounded by", - "must not", - "should not", - "ensure ", - ]; - - pub static OUTPUT_FORMAT: &[&str] = &[ - "json", - "yaml", - "csv", - "xml", - "html", - "markdown", - "table", - "format as", - "in the format", - "output as", - "structure", - "schema", - ]; - - pub static REFERENCE: &[&str] = &[ - "the code above", - "the code below", - "the file", - "the function", - "the api docs", - "the previous", - "earlier you said", - "in the last", - "as shown above", - "above mentioned", - "as discussed", - "you mentioned", - ]; - - pub static NEGATION: &[&str] = &[ - "not ", - "never ", - "neither ", - "except ", - "unless ", - "without ", - "no longer", - "cannot ", - "couldn't ", - "shouldn't ", - "wouldn't ", - "doesn't ", - "don't ", - "didn't ", - ]; - - pub static DOMAIN_SPECIFIC: &[&str] = &[ - "quantum", - "fpga", - "embedded", - "kernel", - "driver", - "genomics", - "blockchain", - "smart contract", - "calculus", - "differential", - "tensor", - "linear algebra", - "signal processing", - "fourier", - "cryptography", - "graph theory", - "topology", - "category theory", - ]; -} diff --git a/crates/switchyard-components/src/dimension_collector/mod.rs b/crates/switchyard-components/src/dimension_collector/mod.rs index ae42f22e..b33b31bb 100644 --- a/crates/switchyard-components/src/dimension_collector/mod.rs +++ b/crates/switchyard-components/src/dimension_collector/mod.rs @@ -1,135 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Context-signal extraction layer. +//! Signal extraction for the stage_router. //! -//! This module owns the pure logic for turning a request's text content -//! into a tuple of scored dimensions plus a token-count estimate and an -//! agentic scalar. The [`crate::request_processors::dimension_collector`] -//! module wraps it as a request-side Switchyard component. -//! -//! Port of `switchyard.experimental.rules_routing.dimension_collector` -//! (deleted in e5f88be2). This layer fits into a broader context-signal -//! vs estimated-signal taxonomy. +//! Owns the pure logic for reading a coding-agent request's tool-call history +//! into a [`ToolResultSignal`] (write/edit/read counts, error severity, streaks), +//! plus response-side signals. The +//! [`crate::request_processors::dimension_collector`] module wraps it as a +//! request-side Switchyard component. -pub mod config; pub mod response; -pub mod scorers; pub mod tool_signals; -pub use config::{DimensionScore, Keywords, ScoringConfig, TokenCountThresholds}; pub use response::{extract_response_signals, ResponseFlag, ResponseSignals}; pub use tool_signals::{ extract_tool_signals, extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, }; - -/// Token-per-character heuristic from ClawRouter (`strategy.ts`): -/// `estimatedTokens = ceil(fullText.length / 4)`. Cheap and good enough -/// for routing decisions. A real tokenizer lands behind a feature flag -/// in future work. -pub const CHARS_PER_TOKEN: u32 = 4; - -/// Estimate token count using the ClawRouter chars/4 heuristic. -pub fn estimate_token_count(text: &str) -> u32 { - let chars = text.chars().count() as u32; - chars.div_ceil(CHARS_PER_TOKEN) -} - -/// Aggregate output of one dimension-collection pass. -/// -/// Stamped into `ProxyContext` by the request-processor adapter so -/// downstream estimators (LLM classifier, rules estimator, …) can read -/// dimension scores without re-extracting them. -/// -/// Phase D removed the previously-emitted `agentic_score` scalar after -/// Calibration showed the underlying signal could not be -/// stably extracted from real TBLite traffic. Downstream consumers that -/// previously read `agentic_score` should use a context-level signal -/// (tool-call count, scope-aware routing input) instead. -#[derive(Clone, Debug, PartialEq)] -pub struct ContextSignals { - pub dimensions: Vec, - pub token_count_estimate: u32, -} - -/// Run all 14 dimension scorers against a prompt. -/// -/// `lower_text` MUST be the lowercased version of the prompt. The raw -/// `prompt` is also accepted so `score_question_complexity` can count -/// `?` against the case-insensitive original (case doesn't matter for -/// `?`, but the unfolded text avoids re-walking the lower_text). -/// -/// Emits a `ContextSignals` carrying: -/// -/// * the 14 dimension scores in the canonical order documented below, -/// * a `chars/4` token-count estimate. -pub fn extract_signals(prompt: &str, lower_text: &str, config: &ScoringConfig) -> ContextSignals { - let token_count_estimate = estimate_token_count(lower_text); - - let dimensions = vec![ - scorers::score_token_count( - token_count_estimate, - config.token_count.short, - config.token_count.long, - ), - scorers::score_code_presence(lower_text, config.code_keywords.as_slice()), - scorers::score_reasoning_markers(lower_text, config.reasoning_keywords.as_slice()), - scorers::score_technical_terms(lower_text, config.technical_keywords.as_slice()), - scorers::score_creative_markers(lower_text, config.creative_keywords.as_slice()), - scorers::score_simple_indicators(lower_text, config.simple_keywords.as_slice()), - scorers::score_imperative_verbs(lower_text, config.imperative_verbs.as_slice()), - scorers::score_constraint_count(lower_text, config.constraint_indicators.as_slice()), - scorers::score_output_format(lower_text, config.output_format_keywords.as_slice()), - scorers::score_reference_complexity(lower_text, config.reference_keywords.as_slice()), - scorers::score_negation_complexity(lower_text, config.negation_keywords.as_slice()), - scorers::score_domain_specificity(lower_text, config.domain_specific_keywords.as_slice()), - scorers::score_multi_step_patterns(lower_text), - scorers::score_question_complexity(prompt), - ]; - - ContextSignals { - dimensions, - token_count_estimate, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn estimate_token_count_uses_chars_div_four_ceiling() { - assert_eq!(estimate_token_count(""), 0); - assert_eq!(estimate_token_count("a"), 1); - assert_eq!(estimate_token_count("abcd"), 1); - assert_eq!(estimate_token_count("abcde"), 2); - } - - #[test] - fn extract_signals_runs_all_fourteen_scorers_in_canonical_order() { - let config = ScoringConfig::default(); - let prompt = "Hello world."; - let signals = extract_signals(prompt, &prompt.to_lowercase(), &config); - - let names: Vec<&str> = signals.dimensions.iter().map(|dim| dim.name).collect(); - assert_eq!( - names, - vec![ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", - ], - ); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/scorers.rs b/crates/switchyard-components/src/dimension_collector/scorers.rs deleted file mode 100644 index 454360ed..00000000 --- a/crates/switchyard-components/src/dimension_collector/scorers.rs +++ /dev/null @@ -1,602 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Pure dimension scorers — `(text, keywords, knobs) -> DimensionScore`. -//! -//! Direct port of the 15 scorer functions in -//! `switchyard.experimental.rules_routing.scorers` (deleted in e5f88be2), -//! itself a port of `ClawRouter/src/router/rules.ts` (MIT). -//! -//! All functions are pure: no shared state, no I/O. The collector -//! lowercases the input text once per request and lowercases the -//! keyword sets once at construction (see [`crate::dimension_collector::Keywords`]), -//! so the per-request hot path is just integer arithmetic and substring scans. - -use super::config::DimensionScore; - -/// Score the `tokenCount` dimension. -/// -/// Below `short` → `-1.0` (looks SIMPLE). Above `long` → `+1.0` -/// (looks COMPLEX). Otherwise `0.0`. Mirrors `scoreTokenCount` in -/// `rules.ts` verbatim. -pub fn score_token_count(estimated_tokens: u32, short: u32, long: u32) -> DimensionScore { - if estimated_tokens < short { - return DimensionScore { - name: "tokenCount", - score: -1.0, - signal: Some(format!("short ({estimated_tokens} tokens)")), - }; - } - if estimated_tokens > long { - return DimensionScore { - name: "tokenCount", - score: 1.0, - signal: Some(format!("long ({estimated_tokens} tokens)")), - }; - } - DimensionScore::zero("tokenCount") -} - -/// Generic keyword-match scorer used by most dimensions. -/// -/// Counts how many `lower_keywords` appear as substrings in `lower_text`. -/// Returns `score_high` once the count reaches `high`, `score_low` once -/// it reaches `low`, otherwise `0.0`. Mirrors `scoreKeywordMatch` in -/// `rules.ts`. -/// -/// Both inputs MUST already be lower-cased. -// -// The 8-arg signature is intentional: this is the shared kernel for 10 -// of the 15 dimension scorers. Each caller is a 1-line specialization -// that pins the per-dimension knobs (name, label, thresholds, scores). -// Bundling into a config struct would force every call site through a -// constructor + clone with no readability win. -#[allow(clippy::too_many_arguments)] -pub fn score_keyword_match( - lower_text: &str, - lower_keywords: &[String], - name: &'static str, - signal_label: &str, - low: usize, - high: usize, - score_low: f32, - score_high: f32, -) -> DimensionScore { - let matches: Vec<&str> = lower_keywords - .iter() - .filter_map(|kw| lower_text.contains(kw.as_str()).then_some(kw.as_str())) - .collect(); - - let count = matches.len(); - if count >= high { - return DimensionScore { - name, - score: score_high, - signal: Some(format!("{signal_label} ({})", preview_matches(&matches),)), - }; - } - if count >= low { - return DimensionScore { - name, - score: score_low, - signal: Some(format!("{signal_label} ({})", preview_matches(&matches),)), - }; - } - DimensionScore::zero(name) -} - -/// Score `codePresence`: how code-shaped does the prompt look? -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 1.0)`. -pub fn score_code_presence(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "codePresence", - "code", - 1, - 2, - 0.5, - 1.0, - ) -} - -/// Score `reasoningMarkers`: how proof / chain-of-thought shaped? -/// -/// Thresholds `(low=1, high=2)`, scores `(0.7, 1.0)`. `score_low` of -/// 0.7 is deliberately high — one reasoning marker is a strong signal. -pub fn score_reasoning_markers(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "reasoningMarkers", - "reasoning", - 1, - 2, - 0.7, - 1.0, - ) -} - -/// Score `technicalTerms` (kubernetes, distributed, algorithm, …). -/// -/// Thresholds `(low=2, high=4)`, scores `(0.5, 1.0)` — needs more -/// matches than other keyword dimensions because technical jargon is -/// noisier and a single hit means less. -pub fn score_technical_terms(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "technicalTerms", - "technical", - 2, - 4, - 0.5, - 1.0, - ) -} - -/// Score `creativeMarkers` (story, poem, brainstorm, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 0.7)` — capped below -/// code / reasoning ceilings because creative requests aren't always -/// complex. -pub fn score_creative_markers(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "creativeMarkers", - "creative", - 1, - 2, - 0.5, - 0.7, - ) -} - -/// Score `simpleIndicators` (what is, hello, define, …). -/// -/// ALWAYS negative: matches push the request toward SIMPLE rather -/// than away from any other tier. Scores `(-1.0, -1.0)`. -pub fn score_simple_indicators(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "simpleIndicators", - "simple", - 1, - 2, - -1.0, - -1.0, - ) -} - -/// Score `imperativeVerbs` (implement, design, refactor, …). -/// -/// Thresholds `(low=2, high=3)`, scores `(0.3, 0.5)` — mild signal; -/// imperative verbs are common across all tiers so weights are low. -/// **Phase B retune**: thresholds raised from `(1, 2)` to `(2, 3)` after -/// Calibration showed a 97% baseline fire rate. Multiple -/// strong verbs now required before the signal fires. -pub fn score_imperative_verbs(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "imperativeVerbs", - "imperative", - 2, - 3, - 0.3, - 0.5, - ) -} - -/// Score `constraintCount` (at most, within, O(), …). -/// -/// Thresholds `(low=1, high=3)`, scores `(0.3, 0.7)` — high score -/// requires multiple constraints; prompts with many constraints tend -/// to be COMPLEX/REASONING-tier algorithmic work. -pub fn score_constraint_count(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "constraintCount", - "constraints", - 1, - 3, - 0.3, - 0.7, - ) -} - -/// Score `outputFormat` (json, yaml, table, csv, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.4, 0.7)` — structured -/// output correlates with structured tasks (parsing, transformation) -/// which skew MEDIUM/COMPLEX. -pub fn score_output_format(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "outputFormat", - "format", - 1, - 2, - 0.4, - 0.7, - ) -} - -/// Score `referenceComplexity` ("the code above", "the API docs", …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.3, 0.5)` — mild signal -/// that the request depends on multi-turn context. -pub fn score_reference_complexity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "referenceComplexity", - "references", - 1, - 2, - 0.3, - 0.5, - ) -} - -/// Score `negationComplexity` (not, never, except, unless, …). -/// -/// Thresholds `(low=2, high=3)`, scores `(0.3, 0.5)` — needs multiple -/// negations to fire; many real prompts contain at least one negation -/// so a single hit doesn't move the needle. -pub fn score_negation_complexity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "negationComplexity", - "negation", - 2, - 3, - 0.3, - 0.5, - ) -} - -/// Score `domainSpecificity` (quantum, FPGA, genomics, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 0.8)` — even one -/// domain-specific term is a strong signal that the request needs a -/// capable model. -pub fn score_domain_specificity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "domainSpecificity", - "domain-specific", - 1, - 2, - 0.5, - 0.8, - ) -} - -/// Score `multiStepPatterns` (`first…then`, `step N`, numbered lists). -/// -/// Three byte-level patterns from `scoreMultiStep` in `rules.ts`, -/// inlined as manual scans to avoid the regex dependency and the -/// runtime fallibility of `Regex::new`. Any hit → score `0.5`; no -/// hit → `0.0`. Caller passes already-lower-cased text. -pub fn score_multi_step_patterns(lower_text: &str) -> DimensionScore { - if has_first_then(lower_text) || has_step_digit(lower_text) || has_numbered_list(lower_text) { - return DimensionScore { - name: "multiStepPatterns", - score: 0.5, - signal: Some("multi-step".to_string()), - }; - } - DimensionScore::zero("multiStepPatterns") -} - -// Match the regex `first.*then`: substring "first" appears somewhere -// before substring "then". -fn has_first_then(lower_text: &str) -> bool { - match lower_text.find("first") { - Some(idx) => lower_text[idx..].contains("then"), - None => false, - } -} - -// Match the regex `step \d`: substring "step " followed by an ASCII digit. -fn has_step_digit(lower_text: &str) -> bool { - let bytes = lower_text.as_bytes(); - bytes - .windows(6) - .any(|window| &window[..5] == b"step " && window[5].is_ascii_digit()) -} - -// Match a tightened numbered-list pattern: ASCII digit, '.', whitespace, -// then an ASCII letter. Phase C retune — the original `\d\.\s` window -// matched version strings ("python 3.11 install"), floating-point -// numbers ("0.5 "), and shell prompts, firing on ~85% of calibration -// prompts. Requiring an ASCII letter after the whitespace catches actual -// list items ("1. Install ...") while ignoring numeric contexts. -fn has_numbered_list(lower_text: &str) -> bool { - let bytes = lower_text.as_bytes(); - bytes.windows(4).any(|window| { - window[0].is_ascii_digit() - && window[1] == b'.' - && window[2].is_ascii_whitespace() - && window[3].is_ascii_alphabetic() - }) -} - -/// Score `questionComplexity` — count of `?` characters in the prompt. -/// -/// `>3` questions → score `0.5` (compound multi-part request); else -/// `0.0`. Mirrors `scoreQuestionComplexity` in `rules.ts`. Caller -/// passes the raw prompt (case doesn't matter for `?`). -pub fn score_question_complexity(prompt: &str) -> DimensionScore { - let count = prompt.matches('?').count(); - if count > 3 { - return DimensionScore { - name: "questionComplexity", - score: 0.5, - signal: Some(format!("{count} questions")), - }; - } - DimensionScore::zero("questionComplexity") -} - -// Phase D: the `score_agentic_task` scorer and the -// `agentic_score` scalar were removed entirely after calibration showed -// the dimension saturating at ~72% even after retuning the keyword list -// to irreversibility markers. No keyword choice discriminated agentic -// turns from "request running inside an agent harness" on real TBLite -// traffic. Downstream estimators that needed an explicit agentic -// signal must rely on context (request shape, tool-call counts) instead. - -/// Inputs to the architecture-settled scorer. -/// -fn preview_matches(matches: &[&str]) -> String { - matches - .iter() - .take(3) - .copied() - .collect::>() - .join(", ") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn token_count_below_short_scores_negative_one() { - let result = score_token_count(20, 50, 500); - assert_eq!(result.score, -1.0); - assert_eq!(result.name, "tokenCount"); - assert!(result.signal.as_deref().unwrap().contains("short")); - } - - #[test] - fn token_count_above_long_scores_positive_one() { - let result = score_token_count(900, 50, 500); - assert_eq!(result.score, 1.0); - assert!(result.signal.as_deref().unwrap().contains("long")); - } - - #[test] - fn token_count_in_band_scores_zero() { - let result = score_token_count(200, 50, 500); - assert_eq!(result.score, 0.0); - assert!(result.signal.is_none()); - } - - #[test] - fn keyword_match_counts_substrings_with_low_high_thresholds() { - let keywords: Vec = ["foo", "bar", "baz"] - .iter() - .map(|s| s.to_string()) - .collect(); - - let none = score_keyword_match( - "nothing relevant here", - &keywords, - "x", - "lbl", - 1, - 2, - 0.5, - 1.0, - ); - assert_eq!(none.score, 0.0); - - let one = score_keyword_match( - "this mentions foo only", - &keywords, - "x", - "lbl", - 1, - 2, - 0.5, - 1.0, - ); - assert_eq!(one.score, 0.5); - - let three = - score_keyword_match("foo and bar and baz", &keywords, "x", "lbl", 1, 2, 0.5, 1.0); - assert_eq!(three.score, 1.0); - } - - #[test] - fn code_presence_fires_on_two_matches() { - let kws: Vec = ["def", "class", "function"] - .iter() - .map(|s| s.to_string()) - .collect(); - let result = score_code_presence("def foo():\n class Bar: ...", &kws); - assert_eq!(result.score, 1.0); - assert_eq!(result.name, "codePresence"); - } - - fn lowered(words: &[&str]) -> Vec { - words.iter().map(|w| w.to_lowercase()).collect() - } - - #[test] - fn reasoning_markers_use_high_low_score_band() { - let kws = lowered(&["prove", "derive", "theorem"]); - assert_eq!(score_reasoning_markers("nothing here", &kws).score, 0.0); - assert_eq!( - score_reasoning_markers("prove the theorem", &kws).score, - 1.0 - ); - assert_eq!(score_reasoning_markers("just prove it", &kws).score, 0.7); - } - - #[test] - fn technical_terms_require_two_matches_to_fire() { - let kws = lowered(&["kubernetes", "distributed", "algorithm", "protocol"]); - // single hit → 0 because low=2 - assert_eq!( - score_technical_terms("a kubernetes question", &kws).score, - 0.0 - ); - assert_eq!( - score_technical_terms("kubernetes and distributed systems", &kws).score, - 0.5 - ); - assert_eq!( - score_technical_terms("kubernetes, distributed algorithm, protocol design", &kws).score, - 1.0 - ); - } - - #[test] - fn creative_markers_capped_below_code_ceiling() { - let kws = lowered(&["story", "poem", "brainstorm"]); - assert_eq!( - score_creative_markers("write a short story", &kws).score, - 0.5 - ); - assert_eq!( - score_creative_markers("write a story and a poem", &kws).score, - 0.7 - ); - } - - #[test] - fn simple_indicators_always_negative() { - let kws = lowered(&["hello", "what is", "define"]); - assert_eq!(score_simple_indicators("hello world", &kws).score, -1.0); - assert_eq!( - score_simple_indicators("hello, define quark", &kws).score, - -1.0 - ); - assert_eq!(score_simple_indicators("hard problem", &kws).score, 0.0); - } - - #[test] - fn imperative_verbs_are_mild_signal() { - let kws = lowered(&["implement", "design", "refactor", "architect"]); - // Phase B thresholds: (low=2, high=3). One match no longer fires. - assert_eq!(score_imperative_verbs("implement a tree", &kws).score, 0.0); - assert_eq!( - score_imperative_verbs("implement and design things", &kws).score, - 0.3 - ); - assert_eq!( - score_imperative_verbs("implement, design, refactor", &kws).score, - 0.5 - ); - } - - #[test] - fn constraint_count_high_requires_three_hits() { - let kws = lowered(&["at most", "within", "no more than"]); - assert_eq!( - score_constraint_count("solve at most quickly", &kws).score, - 0.3 - ); - assert_eq!( - score_constraint_count("at most within no more than", &kws).score, - 0.7 - ); - } - - #[test] - fn output_format_signal_for_structured_keywords() { - let kws = lowered(&["json", "yaml", "csv"]); - assert_eq!(score_output_format("return json", &kws).score, 0.4); - assert_eq!(score_output_format("return json or yaml", &kws).score, 0.7); - } - - #[test] - fn reference_complexity_picks_up_context_pointers() { - let kws = lowered(&["the code above", "the api docs"]); - assert_eq!( - score_reference_complexity("use the code above", &kws).score, - 0.3 - ); - assert_eq!( - score_reference_complexity("the code above and the api docs", &kws).score, - 0.5 - ); - } - - #[test] - fn negation_complexity_requires_two_negations() { - let kws = lowered(&["not", "never", "except", "unless"]); - assert_eq!( - score_negation_complexity("this is not relevant", &kws).score, - 0.0 - ); - assert_eq!( - score_negation_complexity("not now, never tomorrow", &kws).score, - 0.3 - ); - assert_eq!( - score_negation_complexity("not, never, except", &kws).score, - 0.5 - ); - } - - #[test] - fn domain_specificity_strongest_single_keyword_signal() { - let kws = lowered(&["quantum", "fpga", "genomics"]); - assert_eq!( - score_domain_specificity("quantum question", &kws).score, - 0.5 - ); - assert_eq!( - score_domain_specificity("quantum and fpga design", &kws).score, - 0.8 - ); - } - - #[test] - fn multi_step_patterns_match_first_then_step_or_numbered_list() { - assert_eq!( - score_multi_step_patterns("nothing structured here").score, - 0.0 - ); - assert_eq!( - score_multi_step_patterns("first do x, then do y").score, - 0.5 - ); - assert_eq!( - score_multi_step_patterns("follow step 1 carefully").score, - 0.5 - ); - assert_eq!( - score_multi_step_patterns("1. install\n2. configure").score, - 0.5 - ); - } - - #[test] - fn question_complexity_fires_above_three_questions() { - assert_eq!(score_question_complexity("Is this fast?").score, 0.0); - assert_eq!(score_question_complexity("A? B? C? D? E?").score, 0.5); - } -} diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs index 3b34c631..094f58e5 100644 --- a/crates/switchyard-components/src/lib.rs +++ b/crates/switchyard-components/src/lib.rs @@ -12,6 +12,7 @@ pub mod dimension_collector; pub mod intake; pub mod request_processors; pub mod response_processors; +pub mod stage_router; pub mod stats; mod telemetry; @@ -20,8 +21,7 @@ pub use backends::{ MultiLlmBackend, OpenAiNativeBackend, OpenAiPassthroughBackend, StatsLlmBackend, }; pub use dimension_collector::{ - extract_tool_signals, ContextSignals, DimensionScore, Keywords, ResponseFlag, ResponseSignals, - ScoringConfig, ToolResultSignal, + extract_tool_signals, ResponseFlag, ResponseSignals, ToolResultSignal, }; pub use intake::{ HttpIntakeSink, IntakeFormat, IntakePayloadBuilder, IntakeQueueFullPolicy, diff --git a/crates/switchyard-components/src/request_processors/dimension_collector.rs b/crates/switchyard-components/src/request_processors/dimension_collector.rs index f5558c62..617ae1a7 100644 --- a/crates/switchyard-components/src/request_processors/dimension_collector.rs +++ b/crates/switchyard-components/src/request_processors/dimension_collector.rs @@ -1,63 +1,40 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Request-processor adapter for the context-signal extractor. +//! Request-processor adapter for the tool-signal extractor. //! -//! Thin wrapper around [`crate::dimension_collector::extract_signals`]. -//! The pure logic lives in [`crate::dimension_collector`]; this file is -//! the chain entry point — it pulls the user prompt out of `ChatRequest`, -//! lowercases it once, hands off to the extractor, and stamps the -//! resulting [`ContextSignals`] into `ProxyContext` via the typed -//! extensions bag. +//! Thin wrapper around [`crate::dimension_collector::extract_tool_signals_with_window`]. +//! It walks the request's tool-call history and stamps the resulting +//! [`ToolResultSignal`] into `ProxyContext` for the stage_router picker to read. -use serde_json::Value; use switchyard_core::{ChatRequest, ProxyContext, Result}; use crate::dimension_collector::{ - extract_signals, extract_tool_signals_with_window, ContextSignals, ScoringConfig, - ToolResultSignal, DEFAULT_RECENT_WINDOW, + extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, }; -/// Populates `ProxyContext` with scored dimensions extracted from the prompt. -/// -/// Read by downstream estimators (LLM classifier, future rules -/// estimator) via `ctx.get::()`. +/// Populates `ProxyContext` with a [`ToolResultSignal`] read from the request's +/// tool-call history. The stage_router picker reads it via +/// `ctx.get::()`. #[derive(Clone, Debug)] pub struct DimensionCollector { - config: ScoringConfig, recent_window: usize, } impl Default for DimensionCollector { fn default() -> Self { Self { - config: ScoringConfig::default(), recent_window: DEFAULT_RECENT_WINDOW, } } } impl DimensionCollector { - /// Construct a collector with an explicit scoring config and the default - /// `recent_*` window size ([`DEFAULT_RECENT_WINDOW`]). - pub fn new(config: ScoringConfig) -> Self { - Self::with_recent_window(config, DEFAULT_RECENT_WINDOW) - } - - /// Construct a collector with a caller-supplied sliding-window size for - /// `recent_*` signal counts. Smaller windows make the picker more - /// reactive to the very last tool call; larger windows smooth over - /// noisy turn-by-turn fluctuations. - pub fn with_recent_window(config: ScoringConfig, recent_window: usize) -> Self { - Self { - config, - recent_window, - } - } - - /// Returns the underlying scoring config (for tests / introspection). - pub fn config(&self) -> &ScoringConfig { - &self.config + /// Construct a collector with a caller-supplied sliding-window size for the + /// `recent_*` signal counts. Smaller windows make the picker more reactive + /// to the latest tool call; larger windows smooth over turn-by-turn noise. + pub fn with_recent_window(recent_window: usize) -> Self { + Self { recent_window } } /// Returns the configured `recent_*` sliding-window size. @@ -65,132 +42,29 @@ impl DimensionCollector { self.recent_window } - /// Extracts request-side signals and stores them on the request context. + /// Extracts the tool-result signal and stores it on the request context. pub async fn process( &self, ctx: &mut ProxyContext, request: ChatRequest, ) -> Result { - // Text-dimension pass (existing 15 scorers). - let prompt = extract_user_prompt(&request).unwrap_or_default(); - let lower = prompt.to_lowercase(); - let signals = extract_signals(&prompt, &lower, &self.config); - ctx.insert::(signals); - // Tool-signal pass: walk the messages array for tool results and call names. let tool_signal = extract_tool_signals_with_window(&request, self.recent_window); ctx.insert::(tool_signal); Ok(request) } } -/// Extract a single concatenated user-text blob from any inbound `ChatRequest`. -/// -/// Looks at the request body's `messages` array (OpenAI chat / Anthropic -/// messages) or `input` field (OpenAI responses) and concatenates all -/// string user content. Tool calls, role-system messages, and structured -/// content blocks are intentionally ignored — they're either non-user -/// signal or handled by the upcoming dedicated scorers. -fn extract_user_prompt(request: &ChatRequest) -> Option { - let body = request.body(); - let object = body.as_object()?; - - if let Some(messages) = object.get("messages").and_then(Value::as_array) { - let collected = collect_user_text_from_messages(messages); - if !collected.is_empty() { - return Some(collected); - } - } - - if let Some(input) = object.get("input") { - match input { - Value::String(text) => return Some(text.clone()), - Value::Array(items) => { - let collected = collect_user_text_from_messages(items); - if !collected.is_empty() { - return Some(collected); - } - } - _ => {} - } - } - - None -} - -fn collect_user_text_from_messages(messages: &[Value]) -> String { - let mut chunks: Vec = Vec::new(); - for message in messages { - let Some(object) = message.as_object() else { - continue; - }; - let role = object.get("role").and_then(Value::as_str).unwrap_or(""); - if role != "user" { - continue; - } - match object.get("content") { - Some(Value::String(text)) => chunks.push(text.clone()), - Some(Value::Array(parts)) => { - for part in parts { - if let Some(text) = part - .as_object() - .and_then(|p| p.get("text")) - .and_then(Value::as_str) - { - chunks.push(text.to_string()); - } - } - } - _ => {} - } - } - chunks.join("\n") -} - #[cfg(test)] mod tests { use super::*; - use crate::dimension_collector::Keywords; use serde_json::json; #[tokio::test] - async fn stamps_context_signals_into_proxy_context() { - let collector = DimensionCollector::new(ScoringConfig { - code_keywords: Keywords::new(["def", "class"]), - ..ScoringConfig::default() - }); - - let request = ChatRequest::openai_chat(json!({ - "model": "test-model", - "messages": [ - {"role": "system", "content": "you are a helpful assistant"}, - {"role": "user", "content": "def hello(): class Foo: pass"}, - ], - })); - - let mut ctx = ProxyContext::new(); - let _ = collector - .process(&mut ctx, request) - .await - .expect("process ok"); - - let signals = ctx - .get::() - .expect("ContextSignals stamped into context"); - assert_eq!(signals.dimensions.len(), 14); - let code_presence = signals - .dimensions - .iter() - .find(|dim| dim.name == "codePresence") - .expect("codePresence dimension present"); - assert_eq!(code_presence.score, 1.0); - } - - #[tokio::test] - async fn handles_request_with_no_user_content_gracefully() { + async fn stamps_tool_result_signal_into_proxy_context() { let collector = DimensionCollector::default(); let request = ChatRequest::openai_chat(json!({ "model": "test-model", - "messages": [{"role": "system", "content": "only system"}], + "messages": [{"role": "user", "content": "hi"}], })); let mut ctx = ProxyContext::new(); @@ -199,8 +73,6 @@ mod tests { .await .expect("process ok"); - let signals = ctx.get::().expect("signals stamped"); - // Empty user prompt → 0 estimated tokens → tokenCount fires as "short". - assert_eq!(signals.token_count_estimate, 0); + assert!(ctx.get::().is_some()); } } diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs new file mode 100644 index 00000000..cf8c7d83 --- /dev/null +++ b/crates/switchyard-components/src/stage_router.rs @@ -0,0 +1,340 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Stage-router scoring and tier selection — the shared routing core. +//! +//! Decides whether a coding-agent turn goes to the **capable** (strong) or +//! **efficient** (weak) tier from its [`ToolResultSignal`]. Single source of +//! truth, shared by the Rust profile and the Python processor via bindings — +//! only the outer shell differs in how it fetches the decision. +//! +//! Two axes: **error** (`severity`) and **production** (`spinning`/`exploring` +//! push toward capable, `production_intensity` toward efficient). Scored, +//! `tanh`-squashed to `(-1, +1)`, dialed by `confidence_threshold`. See +//! [`pick_tier`] for the decision ladder. + +use crate::dimension_collector::ToolResultSignal; + +/// Turn depth below which stall signals stay quiet — early no-write turns are +/// normal exploration, not a stall. +const STALL_MIN_TURN_DEPTH: u32 = 8; +/// Gain applied before the tanh squash — spreads the small raw weighted sum +/// across the usable confidence range. Without it confidence would cap near +/// ±0.20 and mid/high thresholds would be unreachable. +const SCORE_GAIN: f64 = 5.0; +/// Strongest error severity the scorer sees: critical (`1.0`) is caught by the +/// override, so hard (`0.7`) normalises `severity` to one signal unit. +const HARD_SEVERITY: f64 = 0.7; +/// Weight one maxed signal contributes. Small enough that no single axis pegs +/// the decision; corroboration across the two axes is what raises confidence. +const SIGNAL_UNIT: f64 = 0.10; +/// Critical severity forces the capable tier regardless of the scorer. +const SEVERITY_CRITICAL: f32 = 1.0; + +/// The two tiers a turn can route to. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Tier { + /// Weak / cheap tier. + Efficient, + /// Strong / capable tier. + Capable, +} + +/// Which tier to default to when the scorer is not confident. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PickerMode { + /// Default to capable unless the scorer confidently picks efficient. + CapableFirst, + /// Default to efficient unless the scorer confidently picks capable. + EfficientFirst, +} + +impl PickerMode { + fn default_tier(self) -> Tier { + match self { + Self::CapableFirst => Tier::Capable, + Self::EfficientFirst => Tier::Efficient, + } + } +} + +/// What produced a decision — for stats and explainability. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DecisionSource { + /// Hard override (critical severity or context compaction). + Override, + /// Settled run: recent tests passed with recent production and no error. + TestsPassed, + /// Scorer crossed `confidence_threshold`. + Dimensions, + /// Scorer was not confident; the caller should consult its classifier. + FallOpen, +} + +impl DecisionSource { + /// Stable lowercase label used in stats. + pub fn as_str(self) -> &'static str { + match self { + Self::Override => "override", + Self::TestsPassed => "tests_passed", + Self::Dimensions => "dimensions", + Self::FallOpen => "fall_open", + } + } +} + +/// A signed score in `(-1, +1)` and its magnitude. `confidence == score.abs()`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ScoreResult { + /// Signed score: positive → capable, negative → efficient. + pub score: f64, + /// Decision certainty, `score.abs()`. + pub confidence: f64, +} + +/// The two-axis feature view of a single [`ToolResultSignal`]. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CodingAgentDimensions { + /// Windowed max error severity in `[0, 1]`. + pub severity: f64, + /// `1.0` when a deep turn has no reads, plans, writes, or edits (pure churn). + pub spinning: f64, + /// `1.0` when a deep turn reads/plans but does not write or edit. + pub exploring: f64, + /// Fraction of recent tool ops that produced code (writes + edits). + pub production_intensity: f64, +} + +/// Outcome of [`pick_tier`]: either a resolved decision, or a signal that the +/// caller should consult its (impl-specific, async) classifier. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PickOutcome { + /// The tier was decided without the classifier. + Resolved { + /// The chosen tier. + tier: Tier, + /// What produced it. + source: DecisionSource, + /// Signed scorer value (`0.0` for override / tests-passed). + score: f64, + /// Scorer confidence (`None` where the scorer did not run). + confidence: Option, + }, + /// The scorer was below threshold. The caller runs its classifier; if it + /// has none (or it fails), fall open to `default_tier`. + ConsultClassifier { + /// Signed scorer value, for logging. + score: f64, + /// Scorer confidence, for logging. + confidence: f64, + /// Tier to fall open to when no classifier resolves it. + default_tier: Tier, + }, +} + +/// Project a [`ToolResultSignal`] onto the two-axis dimension space. +pub fn dimensions_from_signal(signal: &ToolResultSignal) -> CodingAgentDimensions { + let recent_ops = signal.recent_write_count + + signal.recent_edit_count + + signal.recent_read_count + + signal.recent_todowrite_count; + let deep_enough = signal.turn_depth >= STALL_MIN_TURN_DEPTH; + let no_production = signal.recent_write_count == 0 && signal.recent_edit_count == 0; + let investigating = signal.recent_read_count >= 1 || signal.recent_todowrite_count >= 1; + // spinning vs exploring partition the "not producing" case by investigative + // activity, so at most one fires — no double-counting on the production axis. + let spinning = deep_enough && no_production && !investigating; + let exploring = deep_enough && no_production && investigating; + + CodingAgentDimensions { + severity: f64::from(signal.severity), + spinning: if spinning { 1.0 } else { 0.0 }, + exploring: if exploring { 1.0 } else { 0.0 }, + production_intensity: ratio( + signal.recent_write_count + signal.recent_edit_count, + recent_ops, + ), + } +} + +/// Score a signal on the two axes, `tanh`-squashed to `(-1, +1)`. +/// +/// One maxed signal reads ~`0.46` confidence, two ~`0.76` — so the threshold +/// dials corroboration: `~0.3` clears on one signal, `~0.5` on ~1.5, `~0.7` on two. +pub fn score_signal(signal: &ToolResultSignal) -> ScoreResult { + let d = dimensions_from_signal(signal); + // Error axis (severity, spinning, exploring → +capable) minus the production + // axis (→ −efficient). Each maxed signal is one SIGNAL_UNIT; severity is + // normalised by its hard cap so it lands there too. + let raw = SIGNAL_UNIT + * (d.severity / HARD_SEVERITY + d.spinning + d.exploring - d.production_intensity); + let score = (SCORE_GAIN * raw).tanh(); + ScoreResult { + score, + confidence: score.abs(), + } +} + +/// Hard **escalate** — force the strong tier no matter what the scorer would +/// say. Fires on a critical error or a compacted context. +fn should_escalate(signal: &ToolResultSignal) -> bool { + // Compaction wipes the accumulated signals, so a task that had escalated + // would snap back to weak — a context big enough to overflow belongs strong. + if signal.compacted { + return true; + } + // A critical error is unambiguous. + signal.severity >= SEVERITY_CRITICAL +} + +/// Hard **de-escalate** — drop to the cheap tier on a settled turn: tests +/// passed, code was just written or edited, and nothing errored in the window. +fn should_deescalate(signal: &ToolResultSignal) -> bool { + signal.tests_passed + && (signal.recent_write_count + signal.recent_edit_count) >= 1 + && signal.severity <= 0.0 +} + +/// Decide a turn's tier from its signal. +/// +/// The rules run in order; the first that fires wins: +/// +/// 1. **Escalate** — a hard reason to go strong (critical error / compaction). +/// 2. **De-escalate** — a hard reason to go cheap (a settled turn). +/// 3. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it. +/// 4. **Fall open** — not confident: hand to the classifier, else the default. +/// +/// Rules 1 and 2 are the two hard shortcuts that skip the scorer — one always +/// escalates, one always de-escalates. **Escalate is checked first**, so a +/// critical error still wins on a turn whose tests also happened to pass. +/// +/// Deterministic and pure: the async classifier lives in the caller, so rule 4 +/// returns [`PickOutcome::ConsultClassifier`] instead of calling it here. The +/// `no_signal` case (no tool activity yet) is handled one level up. +pub fn pick_tier( + signal: &ToolResultSignal, + mode: PickerMode, + confidence_threshold: f64, +) -> PickOutcome { + // 1. Escalate — a hard reason to go strong, ahead of everything else. + if should_escalate(signal) { + return resolved(Tier::Capable, DecisionSource::Override, 0.0, Some(1.0)); + } + + // 2. De-escalate — a hard reason to go cheap (the turn is winding down). + if should_deescalate(signal) { + return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.0, None); + } + + // 3. Scorer — no hard reason either way, so weigh error vs production. If + // confident enough, follow the sign: positive → strong, negative → cheap. + let scored = score_signal(signal); + if scored.confidence >= confidence_threshold { + let tier = if scored.score > 0.0 { + Tier::Capable + } else { + Tier::Efficient + }; + return resolved( + tier, + DecisionSource::Dimensions, + scored.score, + Some(scored.confidence), + ); + } + + // 4. Fall open — the signals didn't corroborate enough to be sure. Hand off + // to the caller's classifier; with none, land on the picker's default. + PickOutcome::ConsultClassifier { + score: scored.score, + confidence: scored.confidence, + default_tier: mode.default_tier(), + } +} + +/// Build a resolved outcome (a decision made without the classifier). +fn resolved( + tier: Tier, + source: DecisionSource, + score: f64, + confidence: Option, +) -> PickOutcome { + PickOutcome::Resolved { + tier, + source, + score, + confidence, + } +} + +fn ratio(numerator: u32, denominator: u32) -> f64 { + if denominator == 0 { + 0.0 + } else { + f64::from(numerator) / f64::from(denominator) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dimension_collector::extract_tool_signals; + use serde_json::json; + use switchyard_core::ChatRequest; + + fn signal_from(messages: serde_json::Value) -> ToolResultSignal { + let request = ChatRequest::openai_chat(json!({"model": "m", "messages": messages})); + extract_tool_signals(&request) + } + + #[test] + fn critical_severity_overrides_to_capable() { + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = SEVERITY_CRITICAL; + match pick_tier(&signal, PickerMode::EfficientFirst, 0.5) { + PickOutcome::Resolved { tier, source, .. } => { + assert_eq!(tier, Tier::Capable); + assert_eq!(source, DecisionSource::Override); + } + other => panic!("expected override, got {other:?}"), + } + } + + #[test] + fn compaction_overrides_to_capable() { + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.compacted = true; + assert!(matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 0.5), + PickOutcome::Resolved { + tier: Tier::Capable, + source: DecisionSource::Override, + .. + } + )); + } + + #[test] + fn one_signal_scores_below_half() { + // A single full wrong signal ≈ 0.46 confidence — just under 0.5. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = HARD_SEVERITY as f32; + let scored = score_signal(&signal); + assert!(scored.score > 0.0); + assert!( + scored.confidence < 0.5, + "one signal should not clear 0.5: {scored:?}" + ); + } + + #[test] + fn quiet_signal_falls_open_to_default() { + let signal = signal_from(json!([{"role": "user", "content": "hi"}])); + match pick_tier(&signal, PickerMode::EfficientFirst, 0.5) { + PickOutcome::ConsultClassifier { default_tier, .. } => { + assert_eq!(default_tier, Tier::Efficient); + } + other => panic!("expected consult-classifier, got {other:?}"), + } + } +} diff --git a/crates/switchyard-py/src/component_bindings.rs b/crates/switchyard-py/src/component_bindings.rs index 9a2c7202..5965811c 100644 --- a/crates/switchyard-py/src/component_bindings.rs +++ b/crates/switchyard-py/src/component_bindings.rs @@ -11,6 +11,7 @@ mod dimension_collector; pub(crate) mod intake; mod request_processors; mod response_processors; +mod stage_router; pub(crate) mod stats; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +22,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { response_processors::register(module)?; backends::register(module)?; dimension_collector::register(module)?; + stage_router::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/component_bindings/dimension_collector.rs b/crates/switchyard-py/src/component_bindings/dimension_collector.rs index 026d193c..8e161ddb 100644 --- a/crates/switchyard-py/src/component_bindings/dimension_collector.rs +++ b/crates/switchyard-py/src/component_bindings/dimension_collector.rs @@ -19,9 +19,8 @@ use pyo3::prelude::*; use pyo3::types::PyList; use switchyard_components::{ dimension_collector::{ - extract_response_signals as core_extract_response_signals, ContextSignals, DimensionScore, - Keywords, ResponseFlag, ResponseSignals, ScoringConfig, ToolResultSignal, - DEFAULT_RECENT_WINDOW, + extract_response_signals as core_extract_response_signals, ResponseFlag, ResponseSignals, + ToolResultSignal, DEFAULT_RECENT_WINDOW, }, DimensionCollector, ResponseSignalCollector, }; @@ -33,161 +32,6 @@ use crate::core_bindings::context::PyProxyContext; use crate::core_bindings::request::PyChatRequest; use crate::core_bindings::response::PyChatResponse; -/// One scorer's output as a Python-visible record. -#[pyclass(name = "DimensionScore", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyDimensionScore { - inner: DimensionScore, -} - -#[pymethods] -impl PyDimensionScore { - #[getter] - fn name(&self) -> &'static str { - self.inner.name - } - - #[getter] - fn score(&self) -> f32 { - self.inner.score - } - - #[getter] - fn signal(&self) -> Option<&str> { - self.inner.signal.as_deref() - } - - fn __repr__(&self) -> String { - format!( - "DimensionScore(name={:?}, score={}, signal={:?})", - self.inner.name, self.inner.score, self.inner.signal, - ) - } -} - -impl PyDimensionScore { - fn from_core(inner: DimensionScore) -> Self { - Self { inner } - } -} - -/// Aggregate context-signal record stamped by [`PyDimensionCollector`]. -#[pyclass(name = "ContextSignals", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyContextSignals { - inner: ContextSignals, -} - -#[pymethods] -impl PyContextSignals { - /// The 15 scored dimensions in canonical order. - #[getter] - fn dimensions(&self, py: Python<'_>) -> PyResult> { - let items: Vec> = self - .inner - .dimensions - .iter() - .map(|dim| Py::new(py, PyDimensionScore::from_core(dim.clone()))) - .collect::>>()?; - Ok(PyList::new(py, items)?.unbind()) - } - - /// Estimated input-token count (`chars / 4` heuristic for now). - #[getter] - fn token_count_estimate(&self) -> u32 { - self.inner.token_count_estimate - } - - fn __repr__(&self) -> String { - format!( - "ContextSignals(dimensions=<{} items>, token_count_estimate={})", - self.inner.dimensions.len(), - self.inner.token_count_estimate, - ) - } -} - -impl PyContextSignals { - fn from_core(inner: ContextSignals) -> Self { - Self { inner } - } -} - -/// Python-facing scoring config; keyword lists are lower-cased on the way in. -#[pyclass(name = "ScoringConfig", skip_from_py_object)] -#[derive(Clone, Debug, Default)] -pub(crate) struct PyScoringConfig { - inner: ScoringConfig, -} - -#[pymethods] -impl PyScoringConfig { - #[new] - #[pyo3(signature = ( - token_count_short = 50, - token_count_long = 500, - code_keywords = vec![], - reasoning_keywords = vec![], - simple_keywords = vec![], - technical_keywords = vec![], - creative_keywords = vec![], - imperative_verbs = vec![], - constraint_indicators = vec![], - output_format_keywords = vec![], - reference_keywords = vec![], - negation_keywords = vec![], - domain_specific_keywords = vec![], - ))] - #[allow(clippy::too_many_arguments)] - fn py_new( - token_count_short: u32, - token_count_long: u32, - code_keywords: Vec, - reasoning_keywords: Vec, - simple_keywords: Vec, - technical_keywords: Vec, - creative_keywords: Vec, - imperative_verbs: Vec, - constraint_indicators: Vec, - output_format_keywords: Vec, - reference_keywords: Vec, - negation_keywords: Vec, - domain_specific_keywords: Vec, - ) -> Self { - let inner = ScoringConfig { - token_count: switchyard_components::dimension_collector::TokenCountThresholds { - short: token_count_short, - long: token_count_long, - }, - code_keywords: Keywords::new(code_keywords), - reasoning_keywords: Keywords::new(reasoning_keywords), - simple_keywords: Keywords::new(simple_keywords), - technical_keywords: Keywords::new(technical_keywords), - creative_keywords: Keywords::new(creative_keywords), - imperative_verbs: Keywords::new(imperative_verbs), - constraint_indicators: Keywords::new(constraint_indicators), - output_format_keywords: Keywords::new(output_format_keywords), - reference_keywords: Keywords::new(reference_keywords), - negation_keywords: Keywords::new(negation_keywords), - domain_specific_keywords: Keywords::new(domain_specific_keywords), - }; - Self { inner } - } - - fn __repr__(&self) -> String { - format!( - "ScoringConfig(token_count_short={}, token_count_long={})", - self.inner.token_count.short, self.inner.token_count.long, - ) - } -} - -impl PyScoringConfig { - fn clone_core(&self) -> ScoringConfig { - self.inner.clone() - } -} - /// Request-side component that runs the dimension collector. #[pyclass(name = "DimensionCollector", skip_from_py_object)] #[derive(Clone, Debug)] @@ -198,12 +42,11 @@ pub(crate) struct PyDimensionCollector { #[pymethods] impl PyDimensionCollector { #[new] - #[pyo3(signature = (config = None, *, recent_window = None))] - fn py_new(config: Option>, recent_window: Option) -> Self { - let scoring = config.map(|cfg| cfg.clone_core()).unwrap_or_default(); + #[pyo3(signature = (*, recent_window = None))] + fn py_new(recent_window: Option) -> Self { let window = recent_window.unwrap_or(DEFAULT_RECENT_WINDOW); Self { - inner: DimensionCollector::with_recent_window(scoring, window), + inner: DimensionCollector::with_recent_window(window), } } @@ -240,18 +83,6 @@ impl PyDimensionCollector { } } -/// Returns the `ContextSignals` stamped by a `DimensionCollector` run. -/// -/// Mirrors the Python idiom `ctx.metadata.get("context_signals")` from -/// the deleted Python implementation, but uses the typed extension bag -/// so consumers don't pay for the dict round-trip. -#[pyfunction] -fn get_context_signals(ctx: PyRef<'_, PyProxyContext>) -> PyResult> { - Ok(ctx - .get_cloned::()? - .map(PyContextSignals::from_core)) -} - /// Closed set of response-side quality flags emitted by the response /// signal collector. Mirrors Rust's /// [`switchyard_components::dimension_collector::ResponseFlag`] enum. @@ -451,18 +282,6 @@ impl PyToolResultSignal { self.inner.severity } - /// Pattern names that fired in the most recent tool result. - #[getter] - fn patterns(&self, py: Python<'_>) -> PyResult> { - let items: Vec> = self - .inner - .patterns - .iter() - .map(|p| Ok(pyo3::types::PyString::new(py, p).unbind())) - .collect::>()?; - Ok(PyList::new(py, items)?.unbind()) - } - /// Consecutive clean tool results at the end of history (``0`` if last failed). #[getter] fn no_error_streak(&self) -> u32 { @@ -535,10 +354,11 @@ impl PyToolResultSignal { self.inner.turn_depth } - /// Character count of the last user message (current-ask size). + /// The request carries a context-compaction summary — the picker forces + holds + /// the strong tier, since compaction otherwise de-escalates the router to weak. #[getter] - fn prompt_char_count(&self) -> u32 { - self.inner.prompt_char_count + fn compacted(&self) -> bool { + self.inner.compacted } fn __repr__(&self) -> String { @@ -557,6 +377,11 @@ impl PyToolResultSignal { fn from_core(inner: ToolResultSignal) -> Self { Self { inner } } + + /// The underlying Rust signal — used by the stage_router picker binding. + pub(crate) fn core(&self) -> &ToolResultSignal { + &self.inner + } } /// Returns the :class:`ToolResultSignal` stamped by a :class:`DimensionCollector` run. @@ -570,15 +395,11 @@ fn get_tool_result_signal(ctx: PyRef<'_, PyProxyContext>) -> PyResult) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; - module.add_function(wrap_pyfunction!(get_context_signals, module)?)?; module.add_function(wrap_pyfunction!(get_response_signals, module)?)?; module.add_function(wrap_pyfunction!(extract_response_signals, module)?)?; module.add_function(wrap_pyfunction!(get_tool_result_signal, module)?)?; diff --git a/crates/switchyard-py/src/component_bindings/stage_router.rs b/crates/switchyard-py/src/component_bindings/stage_router.rs new file mode 100644 index 00000000..95449a16 --- /dev/null +++ b/crates/switchyard-py/src/component_bindings/stage_router.rs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Python binding for the shared stage_router picker. +//! +//! Exposes [`switchyard_components::stage_router::pick_tier`] as +//! `stage_pick_tier(signal, picker_mode, confidence_threshold) -> PickOutcome` +//! so the Python `processor.py` runs the exact same routing decision as the Rust +//! profile. The async classifier and the `no_signal` case stay in Python: this +//! returns a resolved decision, or a request to consult the classifier. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +use switchyard_components::stage_router::{pick_tier, score_signal, PickOutcome, PickerMode, Tier}; + +use super::dimension_collector::PyToolResultSignal; + +fn parse_picker_mode(mode: &str) -> PyResult { + match mode { + "capable_first" => Ok(PickerMode::CapableFirst), + "efficient_first" => Ok(PickerMode::EfficientFirst), + other => Err(PyValueError::new_err(format!( + "unknown picker mode {other:?} (expected capable_first or efficient_first)" + ))), + } +} + +fn tier_str(tier: Tier) -> &'static str { + match tier { + Tier::Capable => "capable", + Tier::Efficient => "efficient", + } +} + +/// Result of [`stage_pick_tier`]. +/// +/// `resolved` is `True` when the picker decided without the classifier — then +/// `tier` and `source` are set. `resolved` is `False` when the scorer was not +/// confident: the caller runs its classifier, and falls back to `default_tier` +/// if it has none or it fails. `score` / `confidence` are always the scorer's. +#[pyclass(name = "PickOutcome", frozen)] +pub(crate) struct PyPickOutcome { + resolved: bool, + tier: Option<&'static str>, + source: Option<&'static str>, + default_tier: &'static str, + score: f64, + confidence: Option, +} + +#[pymethods] +impl PyPickOutcome { + #[getter] + fn resolved(&self) -> bool { + self.resolved + } + + /// Chosen tier (`"capable"` / `"efficient"`) — only when `resolved`. + #[getter] + fn tier(&self) -> Option<&'static str> { + self.tier + } + + /// Decision source (`"override"` / `"tests_passed"` / `"dimensions"`) — only + /// when `resolved`. + #[getter] + fn source(&self) -> Option<&'static str> { + self.source + } + + /// Tier to fall open to when the classifier does not resolve the turn. + #[getter] + fn default_tier(&self) -> &'static str { + self.default_tier + } + + #[getter] + fn score(&self) -> f64 { + self.score + } + + #[getter] + fn confidence(&self) -> Option { + self.confidence + } + + fn __repr__(&self) -> String { + if self.resolved { + format!( + "PickOutcome(resolved, tier={:?}, source={:?}, score={:.3})", + self.tier, self.source, self.score + ) + } else { + format!( + "PickOutcome(consult_classifier, default_tier={:?}, score={:.3})", + self.default_tier, self.score + ) + } + } +} + +/// Decide a turn's tier from its [`ToolResultSignal`], up to (but not including) +/// the classifier. `picker_mode` is `"capable_first"` or `"efficient_first"`. +#[pyfunction] +fn stage_pick_tier( + signal: PyRef<'_, PyToolResultSignal>, + picker_mode: &str, + confidence_threshold: f64, +) -> PyResult { + let mode = parse_picker_mode(picker_mode)?; + let outcome = match pick_tier(signal.core(), mode, confidence_threshold) { + PickOutcome::Resolved { + tier, + source, + score, + confidence, + } => PyPickOutcome { + resolved: true, + tier: Some(tier_str(tier)), + source: Some(source.as_str()), + default_tier: tier_str(tier), + score, + confidence, + }, + PickOutcome::ConsultClassifier { + score, + confidence, + default_tier, + } => PyPickOutcome { + resolved: false, + tier: None, + source: None, + default_tier: tier_str(default_tier), + score, + confidence: Some(confidence), + }, + }; + Ok(outcome) +} + +/// The pure two-axis scorer for a signal, as `(score, confidence)`. Used by +/// offline analysis to replay the raw score independent of the picker mode. +#[pyfunction] +fn stage_score_signal(signal: PyRef<'_, PyToolResultSignal>) -> (f64, f64) { + let result = score_signal(signal.core()); + (result.score, result.confidence) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(stage_pick_tier, module)?)?; + module.add_function(wrap_pyfunction!(stage_score_signal, module)?)?; + Ok(()) +} diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 45c9fe04..be8b6dde 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -21,10 +21,19 @@ Those stages call for different amounts of model capability, which is what the router keys on. For each LLM call, stage-router estimates which stage the agent is in from the -**tool-result history** on the conversation: whether writes and edits are -landing, whether tests pass, whether commands error out, how much read-only -exploration is happening, and how far into the run it is. It turns those signals -into a confidence score for "this turn needs the capable tier", then routes: +**tool-result history** on the conversation, scoring two axes: + +- **WRONG → capable**: `severity` (windowed error severity), `spinning` (deep + churn with no reads or writes), and `exploring` (reading or planning without + producing) push toward the capable tier. +- **PROGRESS → efficient**: `recent_production_intensity` (writes and edits + landing over the recent window) pushes toward the efficient tier. + +The axes are **corroborative**: the signed score is `tanh`-squashed to a +confidence in `[0, 1]`, so one full signal alone scores ~`0.46` and a second +corroborating signal is what pushes it decisively past a `0.5` threshold. A +critical-error severity is a hard override that escalates on its own. The router +then routes: - the **capable** tier for uncertain, exploratory, or error-recovery turns, and - the **efficient** tier for settled, mechanical turns. @@ -89,14 +98,17 @@ clears the threshold to escalate to capable. (If you add the optional classifier, sub-threshold turns go to it instead of staying on the default tier.) -**Start with `0.5`.** It is the default and the recommended starting point. +**Set `0.5` explicitly.** It's the recommended starting point and what the +example below uses. When you omit the field the config default is `0.5` (for +both the profile config and the deprecated route bundle) — but setting it +explicitly keeps the intent clear. | `confidence_threshold` | Include `classifier:` block? | Typical use | |---|---|---| | `0.0` | no | Cost/latency-sensitive. Every signal-based verdict is accepted; no per-turn LLM call. Critical-error signals still escalate to capable. | -| `0.5` | no | Recommended starting point. A single strong signal clears the threshold on its own; derived from SWE-Bench Pro calibration. | +| `0.5` | no | Recommended starting point (config default). The scorer is corroborative — one full wrong signal scores ~`0.46`, just under `0.5` — so a decisive escalation takes a strong signal plus corroboration, while a critical error overrides regardless. Derived from SWE-Bench Pro Python-75 calibration. | | `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. | -| `1.0` | yes (required) | Classifier-driven. Equivalent to classifier-only `coding_agent` routing. | +| `1.0` | yes (required) | Classifier-driven. Equivalent to the legacy `coding_agent` profile. | The signal-vs-classifier split is dataset-dependent. Measure it in production via `routing_decisions.stage_router` on `/v1/stats` rather than relying on @@ -143,33 +155,29 @@ From the overlap tasks (those with both capable and efficient results): **Running the sweep** -Three scripts in `benchmark/calibration/stage_router/` form the pipeline: - -| Script | Input | Output | What it does | -|---|---|---|---| -| `signal_extractor.py` | Harbor task dir (JSONL trajectory) | one signal per turn | Replays a claude-code session, emitting the same signal the stage-router picker would have seen at each turn (write/edit/read counts, severity, tests passed, etc.) | -| `calibrate.py` | Harbor run dirs (one per arm) | `per_task.jsonl`, `per_turn.jsonl` | Reads `result.json` + trajectory JSONL for each task in each arm. Calls `signal_extractor` to build per-turn signals, then writes one record per task (outcome + features) and one record per turn (signal snapshot). Also prints RESCUE/LOSS/SAFE/HARD quadrant counts. | -| `sweep.py` | `per_task.jsonl`, `per_turn.jsonl` | Console table | Replays the per-turn signals through a set of escalation policies and scores each: pass%, escalation rate. The best-scoring policy that keeps escalation rate reasonable is your calibrated threshold. | +Replay your runs through the real Rust scorer and picker with +`benchmark/score_staged_run.py` (the `switchyard-stage-router-scorer` skill). It emits +per-turn scores and per-task routing splits at a given threshold and window — +the actual `pick_capable_first` / `pick_efficient_first` decisions, not a +counterfactual: ```bash -cd benchmark/calibration/stage_router -python calibrate.py \ - --capable-run-dir /tmp/runs/your_capable_run \ - --efficient-run-dir /tmp/runs/your_efficient_probe -python sweep.py +# Score a probe run at a candidate threshold +uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ \ + --threshold 0.5 --window 3 +# → /tmp/-scores.jsonl (per turn: score, confidence, pick_cf, pick_ef) +# → /tmp/-per-task.csv (per task: routing split, mean score/confidence) ``` -`calibrate.py` writes `per_task.jsonl` and `per_turn.jsonl`. `sweep.py` -prints the policy score table; pick the best row from that output. - -Pick the policy whose pass% beats `always_stay` with an acceptable -escalation rate. Translate it to a `confidence_threshold` value. A policy -that escalates ~20% of tasks maps roughly to `confidence_threshold: 0.5` -with `capable_first`. +Sweep a few candidate thresholds and read the routing split and pass rate off +the per-task CSV; the lowest threshold that rescues the RESCUE quadrant without +over-escalating the LOSS quadrant is your calibrated value. Because the scorer +is corroborative, a `0.5` threshold takes ~1.5 signals of agreement — a policy +that escalates ~20% of tasks maps roughly to `confidence_threshold: 0.5` with +`capable_first`. -Even 15–20 probe tasks produce a stable result because signal features are -extracted from the capable-arm trajectories, which are available for all -tasks from the pure-capable run. +Signals come from the actual picker replay, so even 15–20 probe tasks give a +stable result. **Caveat on efficient outcomes in stage-router vs. pure-efficient** @@ -209,6 +217,8 @@ switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` This is the recommended default: routing on tool signals alone, no classifier. +If you omit `confidence_threshold`, the config default of `0.5` applies; the +example sets it explicitly. `fallback_target_on_evict` is required and must reference one of the declared target ids. See [Context-Window Handling](../operations/context_window.md) for @@ -255,8 +265,9 @@ The route counts why each turn was routed the way it was under | Source | When | |---|---| -| `override` | A hard override fired (for example a critical error severity, or a clean test pass). | -| `dimensions` | The signals crossed `confidence_threshold` and picked the tier. | +| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | +| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — dropped the turn to the efficient tier. | +| `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | | `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | | `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured, so the default tier was used. | | `no_signal` | The request arrived before any tool-result history existed. | diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 82e14254..02787f99 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -240,6 +240,9 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "confidence_threshold", "signal_recent_window", "classifier", + "handoff_notes", + "strong_system_prompt", + "weak_system_prompt", "enable_stats", "fallback_target_on_evict", }) diff --git a/switchyard/lib/processors/stage_router/__init__.py b/switchyard/lib/processors/stage_router/__init__.py index 217acc7d..2eadd2fc 100644 --- a/switchyard/lib/processors/stage_router/__init__.py +++ b/switchyard/lib/processors/stage_router/__init__.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""StageRouter — weighted scorer + selective LLM-classifier.""" +"""StageRouter — Rust two-axis scorer + picker, with a selective LLM classifier. + +The routing logic lives in Rust (``switchyard_components::stage_router``); this +package is the Python shell: the picker orchestration, the async classifier, +handoff notes, and decision logging.""" from switchyard.lib.processors.stage_router.classifier import ( CAPABLE_TIER, @@ -13,9 +17,9 @@ DecisionSource, StageRouterDecisionLog, ) -from switchyard.lib.processors.stage_router.dimensions import ( - CodingAgentDimensions, - from_signal, +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_ESCALATION_NOTE, + HandoffNoteInjector, ) from switchyard.lib.processors.stage_router.picker import ( CAPABLE, @@ -23,26 +27,18 @@ pick_capable_first, pick_efficient_first, ) -from switchyard.lib.processors.stage_router.scorer import ( - DEFAULT_WEIGHTS, - ScoreResult, - score, -) __all__ = [ "CONTEXT_KEY", - "DEFAULT_WEIGHTS", + "DEFAULT_ESCALATION_NOTE", "CAPABLE", "CAPABLE_TIER", + "HandoffNoteInjector", "StageRouterDecisionLog", - "CodingAgentDimensions", "DecisionSource", - "ScoreResult", "TierClassifier", "EFFICIENT", "EFFICIENT_TIER", - "from_signal", "pick_capable_first", "pick_efficient_first", - "score", ] diff --git a/switchyard/lib/processors/stage_router/decision_log.py b/switchyard/lib/processors/stage_router/decision_log.py index 88812196..97077b31 100644 --- a/switchyard/lib/processors/stage_router/decision_log.py +++ b/switchyard/lib/processors/stage_router/decision_log.py @@ -8,8 +8,9 @@ from typing import Literal DecisionSource = Literal[ - "override", # _apply_overrides short-circuited (severity ≥ 1.0, large prompt, tests_passed) - "dimensions", # scorer confidence ≥ confidence_threshold + "override", # _apply_overrides short-circuited (severity ≥ 1.0, CRITICAL) + "tests_passed", # settled run (recent test-pass + recent write, no error) → EFFICIENT + "dimensions", # scorer confidence ≥ confidence_threshold → routed by sign "llm-classifier", # classifier consulted and returned a tier "fall_open", # classifier configured but returned None, OR not configured at all "no_signal", # ToolResultSignal not present (first turn) diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py deleted file mode 100644 index ce0fecbe..00000000 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Scorer-ready view of :class:`ToolResultSignal` — all fields normalised to ``[0, 1]``.""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from switchyard_rust.components import ToolResultSignal - - -_PURE_BASH_NORM: float = 8.0 - - -def _saturating(x: float, scale: float) -> float: - """Map non-negative counts to ``[0, 1]``; ``scale`` is the half-saturation point.""" - if x <= 0: - return 0.0 - return 1.0 - math.exp(-x / scale) - - -def _ratio(numerator: int, denominator: int) -> float: - return numerator / denominator if denominator > 0 else 0.0 - - -@dataclass(frozen=True) -class CodingAgentDimensions: - """Normalised, scorer-ready view of a single :class:`ToolResultSignal`.""" - - severity: float - no_error_streak_intensity: float - write_intensity: float - edit_intensity: float - recent_write_intensity: float - planning_active: float - pure_bash_intensity: float - stuck_exploring: float - no_progress: float - tests_passed: float - - -def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: - """Project a :class:`ToolResultSignal` onto the normalised dimension space. - - Note: `read_count` and `turn_depth` are still read from `signal` for the - `stuck_exploring` / `no_progress` boolean gates, but their normalised - intensities aren't exposed as separate dimensions because nothing in - :data:`DEFAULT_WEIGHTS` keys off them. - """ - total_tool_ops = signal.write_count + signal.edit_count + signal.read_count - recent_tool_ops = signal.recent_write_count + signal.recent_edit_count + signal.recent_read_count - stuck = ( - signal.turn_depth >= 8 - and signal.write_count <= 1 - and signal.read_count >= 5 - ) - no_progress = signal.turn_depth > 60 and signal.write_count == 0 - return CodingAgentDimensions( - severity=float(signal.severity), - no_error_streak_intensity=_saturating(signal.no_error_streak, scale=3.0), - write_intensity=_ratio(signal.write_count, total_tool_ops), - edit_intensity=_ratio(signal.edit_count, total_tool_ops), - recent_write_intensity=_ratio(signal.recent_write_count, recent_tool_ops), - planning_active=1.0 if signal.recent_todowrite_count > 0 else 0.0, - pure_bash_intensity=_saturating(signal.pure_bash_streak, scale=_PURE_BASH_NORM), - stuck_exploring=1.0 if stuck else 0.0, - no_progress=1.0 if no_progress else 0.0, - # Only treat tests_passed as a signal once the agent has made real changes; - # early test runs against the unmodified codebase are exploratory, not confirmatory. - tests_passed=1.0 if signal.tests_passed and signal.write_count >= 3 else 0.0, - ) - - -__all__ = ["CodingAgentDimensions", "from_signal"] diff --git a/switchyard/lib/processors/stage_router/handoff_notes.py b/switchyard/lib/processors/stage_router/handoff_notes.py new file mode 100644 index 00000000..375bc997 --- /dev/null +++ b/switchyard/lib/processors/stage_router/handoff_notes.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Optional signal-driven handoff notes for the capable (strong) tier. + +When the stage_router picks the capable tier because a real signal fired, this +injector appends a short, ephemeral guidance note to the outgoing request so the +model knows *why* it was handed the task. + +**Stateless.** No per-session state is tracked. The note fires on every turn +where the capable tier is picked *and* the decision source was a real signal +(``dimensions`` or ``override``). This avoids the stale-state and hash-collision +problems that came with tracking previous tier per session across a shared proxy. + +**Ephemeral by construction.** The note is appended to the request the proxy +forwards for this turn only; it is never written back into the agent's +conversation store. On the next turn the agent re-sends its own history (which +never contained the note) and the injector decides afresh. + +**Truthful escalation gate.** With ``only_on_wrong_signal_escalation`` (default), +the note fires only when the escalation was driven by a real signal +(``override`` — critical severity or compaction — or ``dimensions`` — the scorer +crossing threshold on wrong signals), not by an ambiguous default +(``fall_open`` / ``llm-classifier``). This keeps "the previous model was +stalling" from being asserted when no such signal was seen. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from switchyard.lib.processors.stage_router.picker import CAPABLE, EFFICIENT + +if TYPE_CHECKING: + from switchyard_rust.core import ChatRequest + +log = logging.getLogger(__name__) + +#: Default weak→strong note. Kept to two sentences: state the reason, then the +#: corrective (don't blindly repeat the weak model's last approach). +DEFAULT_ESCALATION_NOTE: str = ( + "[router-guidance] A weaker model was handling this task and showed signs of " + "stalling, looping, or repeated errors on the preceding steps, so control was " + "escalated to you, a stronger model. Re-examine the current state directly and " + "do not simply repeat the previous approach." +) + +#: Default strong→weak note (only used when a deescalation note is configured; +#: the direction is off unless the operator sets one). +DEFAULT_DEESCALATION_NOTE: str = ( + "[router-guidance] A stronger model just completed the hard reasoning for this " + "task; the remaining work is expected to be mechanical. Execute the established " + "plan and avoid re-architecting what is already in place." +) + +#: Decision sources that represent a *real* signal-driven escalation (vs. an +#: ambiguous default). Used by the escalation gate. +_WRONG_ESCALATION_SOURCES: frozenset[str] = frozenset({"override", "dimensions"}) + +class HandoffNoteInjector: + """Appends a guidance note to a tier-picked request driven by real signals. + + Stateless — no per-session tracking. On the capable tier the note fires on + every turn where the decision source was a real wrong signal; on the + efficient tier a de-escalation note fires only when one is configured + (``deescalation_note``), which is off by default. + """ + + def __init__( + self, + *, + escalation_note: str = DEFAULT_ESCALATION_NOTE, + deescalation_note: str | None = None, + only_on_wrong_signal_escalation: bool = True, + ) -> None: + self._escalation_note = escalation_note + self._deescalation_note = deescalation_note + self._only_on_wrong_signal = only_on_wrong_signal_escalation + + def maybe_inject( + self, + request: ChatRequest, + *, + tier: int, + source: str | None, + ) -> bool: + """Inject the guidance note appropriate to the picked tier. + + Capable tier: the escalation note, gated to real wrong signals when + ``only_on_wrong_signal_escalation``. Efficient tier: the de-escalation + note, only when one is configured. Returns ``True`` iff a note was + appended. Never raises — any failure degrades to "no note" so it can't + block routing. + """ + try: + if tier == CAPABLE: + if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: + return False + return _append_note(request, self._escalation_note) + if tier == EFFICIENT and self._deescalation_note is not None: + return _append_note(request, self._deescalation_note) + return False + except Exception: + log.debug("handoff-note injection failed; continuing without note", exc_info=True) + return False + + + +def _append_note(request: ChatRequest, note: str) -> bool: + """Append ``note`` to the request's final user turn. + + Anthropic: add a ``text`` block after the trailing user turn's content + (``tool_result`` blocks must stay first, so the note goes last). OpenAI Chat: + append a fresh trailing ``user`` message (the trailing turn is often a + ``tool`` result, and OpenAI permits a following user message). Both preserve + the cached prefix — only the suffix of the request changes. + """ + body = getattr(request, "body", None) + if not isinstance(body, dict): + return False + messages = body.get("messages") + if not isinstance(messages, list) or not messages: + return False + + fmt = getattr(getattr(request, "request_type", None), "value", None) + block = {"type": "text", "text": note} + + if fmt == "anthropic": + last = messages[-1] + if not isinstance(last, dict) or last.get("role") != "user": + return False + content = last.get("content") + if isinstance(content, list): + content.append(block) + elif isinstance(content, str): + last["content"] = [{"type": "text", "text": content}, block] + else: + return False + else: + # OpenAI Chat / Responses: a trailing user message is the portable append. + messages.append({"role": "user", "content": note}) + + request.replace_body(body) + return True + + +__all__ = [ + "DEFAULT_DEESCALATION_NOTE", + "DEFAULT_ESCALATION_NOTE", + "HandoffNoteInjector", +] diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index ce246129..c144cf3e 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -1,112 +1,91 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Two pickers (capable-first / efficient-first) that share override + scorer -logic; differ only in their fallback tier on low-confidence turns.""" +"""Thin Python shell over the Rust stage_router picker. + +The routing logic — the two-axis scorer, the escalate/de-escalate shortcuts, and +tier selection — lives in Rust (``switchyard_components::stage_router::pick_tier``, +exposed as ``stage_pick_tier``). This module only adds the Python-side pieces the +Rust core deliberately leaves to the caller: the ``no_signal`` case, the async LLM +classifier, and decision-source recording.""" import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from switchyard.lib.processors.stage_router.decision_log import ( CONTEXT_KEY, DecisionSource, StageRouterDecisionLog, ) -from switchyard.lib.processors.stage_router.dimensions import from_signal -from switchyard.lib.processors.stage_router.scorer import DEFAULT_WEIGHTS, score if TYPE_CHECKING: - from collections.abc import Mapping - from switchyard.lib.processors.stage_router.classifier import TierClassifier from switchyard.lib.proxy_context import ProxyContext - from switchyard_rust.components import ToolResultSignal log = logging.getLogger(__name__) EFFICIENT: int = 0 CAPABLE: int = 1 -# Override thresholds — tunable in one place. Promote to YAML if calibration -# diverges across deployments. -#: Force CAPABLE when the latest tool result hit a CRITICAL severity pattern. -SEVERITY_CRITICAL: float = 1.0 -#: Force EFFICIENT when `tests_passed` AND the agent has been working long enough -#: (turn_depth) with few writes — interpreted as the run already settled. -CLEAN_TESTS_MIN_TURN_DEPTH: int = 10 -CLEAN_TESTS_MAX_WRITES: int = 1 +#: Rust returns the tier as a name; map it back to the picker's int constants. +_TIER: dict[str, int] = {"efficient": EFFICIENT, "capable": CAPABLE} async def pick_capable_first( ctx: "ProxyContext", confidence_threshold: float, classifier: "TierClassifier | None" = None, - weights: "Mapping[str, float]" = DEFAULT_WEIGHTS, decision_log: StageRouterDecisionLog | None = None, ) -> int: """CAPABLE default. EFFICIENT only when the scorer is confidently negative.""" - return await _pick( - ctx, - default_tier=CAPABLE, - confidence_threshold=confidence_threshold, - classifier=classifier, - weights=weights, - decision_log=decision_log, - ) + return await _pick(ctx, "capable_first", confidence_threshold, classifier, decision_log) async def pick_efficient_first( ctx: "ProxyContext", confidence_threshold: float, classifier: "TierClassifier | None" = None, - weights: "Mapping[str, float]" = DEFAULT_WEIGHTS, decision_log: StageRouterDecisionLog | None = None, ) -> int: """EFFICIENT default. CAPABLE only when the scorer is confidently positive.""" - return await _pick( - ctx, - default_tier=EFFICIENT, - confidence_threshold=confidence_threshold, - classifier=classifier, - weights=weights, - decision_log=decision_log, - ) + return await _pick(ctx, "efficient_first", confidence_threshold, classifier, decision_log) async def _pick( ctx: "ProxyContext", - default_tier: int, + picker_mode: str, confidence_threshold: float, classifier: "TierClassifier | None", - weights: "Mapping[str, float]", decision_log: StageRouterDecisionLog | None, ) -> int: - from switchyard_rust.components import ( - get_tool_result_signal, # local import: heavy native module + from switchyard_rust.components import ( # local import: heavy native module + get_tool_result_signal, + stage_pick_tier, ) signal = get_tool_result_signal(ctx) if signal is None: - return _record(ctx, decision_log, "no_signal", default_tier) - - override = _apply_overrides(signal) - if override is not None: - return _record(ctx, decision_log, "override", override) - - dimensions = from_signal(signal) - result = score(dimensions, weights=weights) - if result.confidence >= confidence_threshold: - tier = CAPABLE if result.score > 0 else EFFICIENT - return _record(ctx, decision_log, "dimensions", tier) - + default = EFFICIENT if picker_mode == "efficient_first" else CAPABLE + return _record(ctx, decision_log, "no_signal", default) + + # The Rust core runs the escalate/de-escalate shortcuts and the scorer. + outcome = stage_pick_tier(signal, picker_mode, confidence_threshold) + if outcome.resolved: + # resolved ⇒ tier and source are set (guaranteed by the Rust core). + source = cast("DecisionSource", outcome.source) + return _record(ctx, decision_log, source, _TIER[cast(str, outcome.tier)]) + + # Scorer wasn't confident. Consult the classifier; fall open to the default + # tier when there is none or it can't decide. + default = _TIER[outcome.default_tier] if classifier is None: - return _record(ctx, decision_log, "fall_open", default_tier) + return _record(ctx, decision_log, "fall_open", default) verdict = await classifier.classify(ctx, signal) if verdict == "capable": return _record(ctx, decision_log, "llm-classifier", CAPABLE) if verdict == "efficient": return _record(ctx, decision_log, "llm-classifier", EFFICIENT) - return _record(ctx, decision_log, "fall_open", default_tier) + return _record(ctx, decision_log, "fall_open", default) def _record( @@ -126,17 +105,4 @@ def _record( return tier -def _apply_overrides(signal: "ToolResultSignal") -> int | None: - """Non-negotiable, signal-derived shortcuts that bypass the scorer.""" - if signal.severity >= SEVERITY_CRITICAL: - return CAPABLE - if ( - signal.tests_passed - and signal.turn_depth >= CLEAN_TESTS_MIN_TURN_DEPTH - and signal.write_count <= CLEAN_TESTS_MAX_WRITES - ): - return EFFICIENT - return None - - __all__ = ["CAPABLE", "EFFICIENT", "pick_capable_first", "pick_efficient_first"] diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py deleted file mode 100644 index 0c3763ad..00000000 --- a/switchyard/lib/processors/stage_router/scorer.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Weighted linear scorer: signed score in ``[-1, +1]``, confidence = ``abs(score)``.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field - -from switchyard.lib.processors.stage_router.dimensions import CodingAgentDimensions - -#: Default linear weights. Positive ⇒ CAPABLE; negative ⇒ EFFICIENT. Calibrated so -#: a single high-impact axis lands past the 0.5 default confidence threshold. -DEFAULT_WEIGHTS: Mapping[str, float] = { - "severity": 0.80, - "stuck_exploring": 0.70, - "no_progress": 0.60, - "tests_passed": -0.80, - "planning_active": -0.70, - "write_intensity": -0.40, - "edit_intensity": -0.30, - "recent_write_intensity": -0.30, - "pure_bash_intensity": -0.30, - "no_error_streak_intensity": -0.20, -} - - -@dataclass(frozen=True) -class ScoreResult: - """Output of :func:`score`. ``confidence == abs(score)`` by construction.""" - - score: float - confidence: float - contributions: Mapping[str, float] = field(default_factory=dict) - - -def score( - dimensions: CodingAgentDimensions, - *, - weights: Mapping[str, float] = DEFAULT_WEIGHTS, -) -> ScoreResult: - """Score ``dimensions`` against ``weights``; raw sum is clipped to ``[-1, +1]``.""" - contributions: dict[str, float] = {} - raw = 0.0 - for field_name, weight in weights.items(): - value = getattr(dimensions, field_name, 0.0) - contribution = value * weight - contributions[field_name] = contribution - raw += contribution - clipped = max(-1.0, min(1.0, raw)) - return ScoreResult(score=clipped, confidence=abs(clipped), contributions=contributions) - - -__all__ = ["DEFAULT_WEIGHTS", "ScoreResult", "score"] diff --git a/switchyard/lib/processors/stage_router_request_processor.py b/switchyard/lib/processors/stage_router_request_processor.py index b91c02d4..1edc0fa7 100644 --- a/switchyard/lib/processors/stage_router_request_processor.py +++ b/switchyard/lib/processors/stage_router_request_processor.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from switchyard.lib.backends.llm_target import LlmTarget + from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest @@ -48,6 +49,9 @@ def __init__( picker: TierPicker, classifier: TierClassifier | None = None, decision_log: StageRouterDecisionLog | None = None, + handoff_injector: HandoffNoteInjector | None = None, + strong_system_prompt: str | None = None, + weak_system_prompt: str | None = None, ) -> None: if len(targets) != 2: raise ValueError(f"stage_router requires exactly 2 targets, got {len(targets)}") @@ -57,6 +61,9 @@ def __init__( self._classifier = classifier self._max_index = len(targets) - 1 self._decision_log = decision_log if decision_log is not None else StageRouterDecisionLog() + self._handoff_injector = handoff_injector + self._strong_system_prompt = strong_system_prompt + self._weak_system_prompt = weak_system_prompt self._stats_accumulator: StatsAccumulator | None = None def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None: @@ -83,6 +90,17 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: ctx.selected_target = self._target_ids[idx] ctx.selected_model = self._target_models[idx] await self._record_decision_source(ctx) + if self._handoff_injector is not None: + source = ctx.metadata.get(CONTEXT_KEY) + self._handoff_injector.maybe_inject( + request, + tier=idx, + source=source if isinstance(source, str) else None, + ) + if self._strong_system_prompt is not None and idx == CAPABLE: + _prepend_system_prompt(request, self._strong_system_prompt) + if self._weak_system_prompt is not None and idx == EFFICIENT: + _prepend_system_prompt(request, self._weak_system_prompt) log.debug( "stage_router pick: idx=%d target=%s model=%s", idx, ctx.selected_target, ctx.selected_model, @@ -113,6 +131,37 @@ async def _record_decision_source(self, ctx: ProxyContext) -> None: log.debug("failed to record stage_router decision source", exc_info=True) +def _prepend_system_prompt(request: ChatRequest, prompt: str) -> None: + """Prepend ``prompt`` to the request's system content. Never raises.""" + try: + body = getattr(request, "body", None) + if not isinstance(body, dict): + return + fmt = getattr(getattr(request, "request_type", None), "value", None) + if fmt == "anthropic": + # Anthropic: top-level "system" field (string or content-block list) + existing = body.get("system", "") + if isinstance(existing, str): + body["system"] = prompt + ("\n\n" + existing if existing else "") + elif isinstance(existing, list): + body["system"] = [{"type": "text", "text": prompt}] + existing + else: + body["system"] = prompt + else: + # OpenAI: prepend a system message before existing messages + messages = body.get("messages") + if not isinstance(messages, list): + return + if messages and messages[0].get("role") == "system": + existing = messages[0].get("content", "") + messages[0] = {"role": "system", "content": prompt + "\n\n" + existing} + else: + messages.insert(0, {"role": "system", "content": prompt}) + request.replace_body(body) + except Exception: + log.debug("strong_system_prompt injection failed; continuing without", exc_info=True) + + __all__ = [ "BUILTIN_PICKERS", "CAPABLE", diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py index 8429832f..32a0b4e9 100644 --- a/switchyard/lib/profiles/stage_router.py +++ b/switchyard/lib/profiles/stage_router.py @@ -10,6 +10,7 @@ from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint from switchyard.lib.processors.stage_router import StageRouterDecisionLog, TierClassifier +from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector from switchyard.lib.processors.stage_router_request_processor import ( BUILTIN_PICKERS, StageRouterRequestProcessor, @@ -18,6 +19,7 @@ from switchyard.lib.profiles.chain import ComponentChainProfile from switchyard.lib.profiles.stage_router_config import ( ClassifierConfig, + HandoffNoteConfig, StageRouterConfig, ) from switchyard.lib.profiles.table import profile_config @@ -53,6 +55,9 @@ def build(self) -> ComponentChainProfile: picker=_build_tier_picker(config, decision_log, classifier), classifier=classifier, decision_log=decision_log, + handoff_injector=_build_handoff_injector(config.handoff_notes), + strong_system_prompt=config.strong_system_prompt, + weak_system_prompt=config.weak_system_prompt, ) ) @@ -83,6 +88,17 @@ def _build_tier_picker( ) +def _build_handoff_injector(config: HandoffNoteConfig | None) -> HandoffNoteInjector | None: + """Build the optional tier-transition note injector; ``None`` when disabled.""" + if config is None or not config.enabled: + return None + return HandoffNoteInjector( + escalation_note=config.escalation_note, + deescalation_note=config.deescalation_note, + only_on_wrong_signal_escalation=config.only_on_wrong_signal_escalation, + ) + + def _build_classifier(config: ClassifierConfig | None) -> TierClassifier | None: """Build the optional LLM fallback classifier for stage_router routing.""" if config is None: diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index ed90e321..0092bb59 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -10,6 +10,9 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_ESCALATION_NOTE, +) #: Picker mode accepted in YAML. The profile resolves it to a :class:`TierPicker`. #: The name describes the *default tier* — what the picker returns when the @@ -34,6 +37,31 @@ class ClassifierConfig(BaseModel): recent_turn_window: int = Field(default=3, ge=0) +class HandoffNoteConfig(BaseModel): + """Optional tier-transition guidance notes injected into the request. + + Off by default. When ``enabled``, a short ephemeral note is appended to the + outgoing request on a *tier transition* only (weak↔strong), telling the model + taking over why it was handed the task. Notes are never persisted into the + agent's history, so they do not accumulate across turns. See + ``switchyard.lib.processors.stage_router.handoff_notes``. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + #: Master switch. When ``False`` (default) no note is ever injected. + enabled: bool = False + #: Note injected on every capable-tier turn driven by a real wrong signal. + escalation_note: str = DEFAULT_ESCALATION_NOTE + #: Optional strong→weak note. When set, injected on every efficient-tier turn + #: (hand-back to the weak model); ``None`` (default) leaves de-escalation off. + deescalation_note: str | None = None + #: When ``True`` (default) only inject when the decision source was a real + #: signal (``dimensions`` or ``override``), not an ambiguous default + #: (``fall_open``). Keeps "stalling" from being asserted without evidence. + only_on_wrong_signal_escalation: bool = True + + class StageRouterConfig(BaseModel): """Configuration for the stage-router-routing profile.""" @@ -58,6 +86,16 @@ class StageRouterConfig(BaseModel): #: ``DEFAULT_RECENT_WINDOW`` in ``tool_signals.rs``. signal_recent_window: int = Field(default=3, ge=1) classifier: ClassifierConfig | None = None + #: Optional tier-transition handoff notes (off unless ``handoff_notes.enabled``). + handoff_notes: HandoffNoteConfig | None = None + #: Optional system prompt prepended to every request routed to the capable + #: (strong) tier. Injected before the model sees the request — useful for + #: conciseness guidance that should apply on every Opus turn without + #: polluting the agent's conversation history. + strong_system_prompt: str | None = None + #: Optional system prompt prepended to every request routed to the efficient + #: (weak) tier. Useful for conciseness/bail-early guidance on Nemotron turns. + weak_system_prompt: str | None = None enable_stats: bool = True @field_validator("capable", "efficient", mode="before") diff --git a/switchyard_rust/components.py b/switchyard_rust/components.py index 74f86864..f7578950 100644 --- a/switchyard_rust/components.py +++ b/switchyard_rust/components.py @@ -13,9 +13,7 @@ { "AnthropicNativeBackend", "BackendFormat", - "ContextSignals", "DimensionCollector", - "DimensionScore", "EndpointConfig", "IntakeQueueFullPolicy", "IntakeRequestMetadata", @@ -27,31 +25,30 @@ "MultiLlmBackend", "OpenAiNativeBackend", "OpenAiPassthroughBackend", + "PickOutcome", "RandomRoutingProcessorConfig", "RequestMetadata", "ResponseFlag", "ResponseSignalCollector", "ResponseSignals", - "ScoringConfig", "StatsAccumulator", "StatsLlmBackend", "StatsRequestProcessor", "StatsResponseProcessor", "ToolResultSignal", "extract_response_signals", - "get_context_signals", "get_response_signals", "get_tool_result_signal", "set_stats_route_label", + "stage_pick_tier", + "stage_score_signal", } ) if TYPE_CHECKING: AnthropicNativeBackend: type[Any] BackendFormat: type[Any] - ContextSignals: type[Any] DimensionCollector: type[Any] - DimensionScore: type[Any] EndpointConfig: type[Any] IntakeQueueFullPolicy: type[Any] IntakeRequestMetadata: type[Any] @@ -63,22 +60,23 @@ MultiLlmBackend: type[Any] OpenAiNativeBackend: type[Any] OpenAiPassthroughBackend: type[Any] + PickOutcome: type[Any] RandomRoutingProcessorConfig: type[Any] RequestMetadata: type[Any] ResponseFlag: type[Any] ResponseSignalCollector: type[Any] ResponseSignals: type[Any] - ScoringConfig: type[Any] StatsAccumulator: type[Any] StatsLlmBackend: type[Any] StatsRequestProcessor: type[Any] StatsResponseProcessor: type[Any] ToolResultSignal: type[Any] extract_response_signals: Any - get_context_signals: Any get_response_signals: Any get_tool_result_signal: Any set_stats_route_label: Any + stage_pick_tier: Any + stage_score_signal: Any def __getattr__(name: str) -> object: diff --git a/switchyard_rust/components.pyi b/switchyard_rust/components.pyi index 01621be5..dc2b76a6 100644 --- a/switchyard_rust/components.pyi +++ b/switchyard_rust/components.pyi @@ -286,40 +286,9 @@ class IntakeResponseProcessor: async def shutdown(self) -> None: ... -class DimensionScore: - name: str - score: float - signal: str | None - - -class ContextSignals: - dimensions: list[DimensionScore] - token_count_estimate: int - - -class ScoringConfig: - def __init__( - self, - token_count_short: int = 50, - token_count_long: int = 500, - code_keywords: Iterable[str] = (), - reasoning_keywords: Iterable[str] = (), - simple_keywords: Iterable[str] = (), - technical_keywords: Iterable[str] = (), - creative_keywords: Iterable[str] = (), - imperative_verbs: Iterable[str] = (), - constraint_indicators: Iterable[str] = (), - output_format_keywords: Iterable[str] = (), - reference_keywords: Iterable[str] = (), - negation_keywords: Iterable[str] = (), - domain_specific_keywords: Iterable[str] = (), - ) -> None: ... - - class DimensionCollector: def __init__( self, - config: ScoringConfig | None = None, *, recent_window: int | None = None, ) -> None: ... @@ -328,12 +297,8 @@ class DimensionCollector: async def shutdown(self) -> None: ... -def get_context_signals(ctx: ProxyContext) -> ContextSignals | None: ... - - class ToolResultSignal: severity: float - patterns: tuple[str, ...] turn_depth: int write_count: int edit_count: int @@ -346,12 +311,29 @@ class ToolResultSignal: pure_bash_streak: int no_error_streak: int tests_passed: bool - prompt_char_count: int + compacted: bool def get_tool_result_signal(ctx: ProxyContext) -> ToolResultSignal | None: ... +class PickOutcome: + resolved: bool + tier: str | None + source: str | None + default_tier: str + score: float + confidence: float | None + + +def stage_pick_tier( + signal: ToolResultSignal, picker_mode: str, confidence_threshold: float +) -> PickOutcome: ... + + +def stage_score_signal(signal: ToolResultSignal) -> tuple[float, float]: ... + + class ResponseFlag: MALFORMED_TOOL_CALL_JSON: ClassVar[ResponseFlag] EMPTY_RESPONSE: ClassVar[ResponseFlag] diff --git a/tests/test_dimension_collector.py b/tests/test_dimension_collector.py deleted file mode 100644 index 0e10b506..00000000 --- a/tests/test_dimension_collector.py +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end Python tests for the Rust-backed ``DimensionCollector``. - -Mirrors the coverage of the deleted -``switchyard/experimental/rules_routing/tests/test_dimension_collector.py`` -against the new Rust port exposed through ``switchyard_rust.components``. -""" - -from __future__ import annotations - -from typing import Any, cast - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.components import ( - ContextSignals, - DimensionCollector, - DimensionScore, - ScoringConfig, - get_context_signals, -) -from switchyard_rust.core import ChatRequest - - -def _request(prompt: str) -> ChatRequest: - return ChatRequest.openai_chat( - cast( - Any, - { - "model": "client-model", - "messages": [{"role": "user", "content": prompt}], - }, - ), - ) - - -def _default_config() -> ScoringConfig: - return ScoringConfig( - code_keywords=["def", "class", "function"], - reasoning_keywords=["prove", "theorem", "derive"], - simple_keywords=["hello", "what is", "define"], - technical_keywords=["kubernetes", "distributed", "algorithm", "protocol"], - creative_keywords=["story", "poem", "brainstorm"], - imperative_verbs=["build", "create", "implement"], - constraint_indicators=["at most", "within", "no more than"], - output_format_keywords=["json", "yaml", "csv"], - reference_keywords=["the code above", "the api docs"], - negation_keywords=["not", "never", "except", "unless"], - domain_specific_keywords=["quantum", "fpga", "genomics"], - ) - - -async def test_dimension_collector_stamps_context_signals_into_proxy_context() -> None: - """Hot-path assertion: ``get_context_signals`` returns the stamped record.""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("def hello(): class Foo: pass")) - - signals = get_context_signals(ctx) - assert isinstance(signals, ContextSignals) - assert len(signals.dimensions) == 14 - code_presence = next(d for d in signals.dimensions if d.name == "codePresence") - assert isinstance(code_presence, DimensionScore) - assert code_presence.score == 1.0 - - -async def test_dimensions_are_emitted_in_canonical_order() -> None: - """Order is a public contract for downstream estimators (e.g. weighted sums).""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("Hello world.")) - - signals = get_context_signals(ctx) - assert signals is not None - assert [d.name for d in signals.dimensions] == [ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", - ] - - -async def test_token_count_short_pushes_negative_score() -> None: - """A bare ``hello`` should score short (chars/4 = 1 ≪ default short=50).""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("hello")) - - signals = get_context_signals(ctx) - assert signals is not None - token_dim = next(d for d in signals.dimensions if d.name == "tokenCount") - assert token_dim.score == -1.0 - - -async def test_simple_indicators_pull_negative_for_chitchat() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("hello, what is the meaning of life?")) - - signals = get_context_signals(ctx) - assert signals is not None - simple_dim = next(d for d in signals.dimensions if d.name == "simpleIndicators") - assert simple_dim.score == -1.0 - - -async def test_multi_step_patterns_detect_first_then_chain() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("first do x then do y")) - - signals = get_context_signals(ctx) - assert signals is not None - multi = next(d for d in signals.dimensions if d.name == "multiStepPatterns") - assert multi.score == 0.5 - - -async def test_question_complexity_fires_above_three_questions() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("A? B? C? D? E?")) - - signals = get_context_signals(ctx) - assert signals is not None - questions = next(d for d in signals.dimensions if d.name == "questionComplexity") - assert questions.score == 0.5 - - -async def test_collector_without_args_uses_populated_rust_defaults() -> None: - """No-arg `DimensionCollector()` picks up `ScoringConfig::default()`. - - The Rust-side `Default` ships populated keyword lists. A bland prompt like ``"just some text"`` should still - emit the full 14-dimension vector with sensible zeros for - non-matching scorers. - """ - collector = DimensionCollector() - ctx = ProxyContext() - - await collector.process(ctx, _request("just some text")) - - signals = get_context_signals(ctx) - assert signals is not None - assert len(signals.dimensions) == 14 - # `"just some text"` is short (< 50 token estimate) so `tokenCount` - # fires negative; no keyword scorer should hit on this prompt. - by_name = {d.name: d.score for d in signals.dimensions} - assert by_name["tokenCount"] == -1.0 - keyword_dims = { - "codePresence", "reasoningMarkers", "technicalTerms", - "creativeMarkers", "simpleIndicators", "imperativeVerbs", - "constraintCount", "outputFormat", "referenceComplexity", - "negationComplexity", "domainSpecificity", - } - for name in keyword_dims: - assert by_name[name] == 0.0, f"{name} unexpectedly fired" - - -async def test_get_context_signals_returns_none_when_not_stamped() -> None: - """Reader must not fabricate a record when no collector has run.""" - ctx = ProxyContext() - assert get_context_signals(ctx) is None diff --git a/tests/test_stage_router_handoff_notes.py b/tests/test_stage_router_handoff_notes.py new file mode 100644 index 00000000..7afc4b97 --- /dev/null +++ b/tests/test_stage_router_handoff_notes.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the optional handoff-note injector. + +The injector is stateless: on the capable tier it appends an escalation note +when the decision source is a real wrong signal; on the efficient tier it +appends a de-escalation note only when one is configured. +""" + +from __future__ import annotations + +from switchyard.lib.processors.stage_router import CAPABLE, EFFICIENT +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_DEESCALATION_NOTE, + DEFAULT_ESCALATION_NOTE, + HandoffNoteInjector, +) +from switchyard_rust.core import ChatRequest + + +def _anthropic(task: str = "solve the task", turn: str = "next") -> ChatRequest: + """Anthropic request whose trailing user turn carries a tool_result block.""" + return ChatRequest.anthropic({ + "system": "you are a coding agent", + "messages": [ + {"role": "user", "content": task}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": turn}]}, + ], + }) + + +def _last_user_text(request: ChatRequest) -> str: + content = request.body["messages"][-1]["content"] + if isinstance(content, str): + return content + return "".join(b.get("text", "") for b in content if isinstance(b, dict)) + + +def test_escalation_injects_note_as_trailing_block(): + inj = HandoffNoteInjector() + req = _anthropic() + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True + content = req.body["messages"][-1]["content"] + # tool_result stays first; the note is appended after it as a text block. + assert content[0]["type"] == "tool_result" + assert content[-1] == {"type": "text", "text": DEFAULT_ESCALATION_NOTE} + + +def test_escalation_from_ambiguous_source_gated_off_by_default(): + inj = HandoffNoteInjector() + req = _anthropic() + # capable_first can escalate on fall_open (ambiguous default) — the truthful + # gate suppresses the "prior model stalled" note there. + assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is False + assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) + + +def test_wrong_signal_gate_can_be_disabled(): + inj = HandoffNoteInjector(only_on_wrong_signal_escalation=False) + req = _anthropic() + assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is True + + +def test_override_source_counts_as_wrong_signal(): + """Compaction / critical-severity escalations stamp ``override`` → note fires.""" + inj = HandoffNoteInjector() + req = _anthropic() + assert inj.maybe_inject(req, tier=CAPABLE, source="override") is True + + +def test_deescalation_off_by_default(): + inj = HandoffNoteInjector() + req = _anthropic() + # No de-escalation note configured → hand-back to the weak tier injects nothing. + assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is False + assert DEFAULT_DEESCALATION_NOTE not in _last_user_text(req) + + +def test_deescalation_note_injected_when_configured(): + inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) + req = _anthropic() + assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is True + assert DEFAULT_DEESCALATION_NOTE in _last_user_text(req) + + +def test_notes_do_not_accumulate_across_turns(): + """Each injected request holds at most one note — nothing carries over.""" + inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) + up = _anthropic(turn="b") + inj.maybe_inject(up, tier=CAPABLE, source="dimensions") # escalate + down = _anthropic(turn="c") + inj.maybe_inject(down, tier=EFFICIENT, source="dimensions") # de-escalate + # Each request is built fresh, so the de-escalation request carries only the + # de-escalation note, never the escalation one. + text = _last_user_text(down) + assert DEFAULT_DEESCALATION_NOTE in text + assert DEFAULT_ESCALATION_NOTE not in text + + +def test_openai_chat_appends_trailing_user_message(): + inj = HandoffNoteInjector() + req = ChatRequest.openai_chat({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "", "tool_calls": [ + {"id": "t1", "type": "function", + "function": {"name": "Bash", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "t1", "content": "err"}, + ], + }) + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True + last = req.body["messages"][-1] + assert last["role"] == "user" + assert last["content"] == DEFAULT_ESCALATION_NOTE diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py index 3fa6f3fd..dc8a49f8 100644 --- a/tests/test_stage_router_pickers.py +++ b/tests/test_stage_router_pickers.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Behavioural tests for the stage_router pickers.""" +"""Behavioural tests for the stage_router pickers. + +The two pickers share every decision path (override → tests_passed → scorer → +classifier) and differ only in the low-confidence default tier. +""" from dataclasses import dataclass from typing import Any @@ -34,6 +38,11 @@ def _msg_tool_call(name: str) -> dict: return {"role": "assistant", "tool_calls": [{"function": {"name": name, "arguments": "{}"}}]} +def _msg_bash(cmd: str) -> dict: + args = f'{{"command": "{cmd}"}}' + return {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": args}}]} + + def _msg_tool_result(content: str) -> dict: return {"role": "tool", "tool_call_id": "x", "content": content} @@ -71,11 +80,7 @@ async def acompletion( def _stub_classifier(tier: str | None) -> TierClassifier: - return TierClassifier( - model="stub", - api_key="stub", - client=_StubLLMClient(tier), - ) + return TierClassifier(model="stub", api_key="stub", client=_StubLLMClient(tier)) @pytest.mark.asyncio @@ -85,15 +90,89 @@ async def test_critical_severity_overrides_to_capable(): {"role": "user", "content": "retry"}, ]) assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx([ + _msg_tool_result("Out of memory: cannot allocate memory"), + {"role": "user", "content": "retry"}, + ]) assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE @pytest.mark.asyncio -async def test_tests_passed_with_edits_drops_to_efficient(): +async def test_compaction_overrides_to_capable(): + """A compacted context forces CAPABLE for both pickers, even with a production + signal that would otherwise route EFFICIENT — a task hard enough to overflow its + context belongs on strong, and the override holds it there.""" + msgs = [ + {"role": "user", "content": + "This session is being continued from a previous conversation that ran out of context."}, + ] + [_msg_tool_call("Write"), _msg_tool_result("ok")] * 3 + ctx = await _ctx(msgs) + assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx(msgs) + assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + + +@pytest.mark.asyncio +async def test_tests_passed_with_edits_routes_to_efficient(): + """A settled run (recent test-pass + recent edit, no error) is safe on cheap tier.""" messages = [_msg_tool_call("Edit")] * 3 + [_msg_tool_result("all tests passed")] * 3 messages += [{"role": "user", "content": "ok continue"}] * 12 ctx = await _ctx(messages) assert await pick_capable_first(ctx, confidence_threshold=0.7) == EFFICIENT + ctx = await _ctx(messages) + assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT + + +@pytest.mark.asyncio +async def test_tests_passed_blocked_by_windowed_error(): + """A HARD error in the window blocks the settled-run shortcut (finding G) — the + turn falls through to the scorer instead of being swallowed as EFFICIENT.""" + log = StageRouterDecisionLog() + messages = [_msg_tool_call("Edit")] * 3 + [ + _msg_tool_result("all tests passed"), + _msg_tool_result("Traceback (most recent call last):\n ValueError: bad"), + ] + ctx = await _ctx(messages) + await pick_capable_first(ctx, confidence_threshold=0.2, decision_log=log) + assert ctx.metadata[CONTEXT_KEY] != "tests_passed" + + +@pytest.mark.asyncio +async def test_dimensions_routes_capable_on_corroborated_wrong_signals(): + """Deep pure-command cycling (spinning) that ends in an error corroborates + severity + spinning → CAPABLE by sign, for both pickers, no classifier needed.""" + log = StageRouterDecisionLog() + classifier = _stub_classifier(tier="efficient") # would flip if consulted + seq: list[dict] = [] + for i in range(5): + seq.append(_msg_bash("make")) + seq.append(_msg_tool_result( + "Traceback (most recent call last):\n ValueError" if i == 4 else "building..." + )) + seq.append({"role": "user", "content": "next"}) + ctx = await _ctx(seq) + tier = await pick_capable_first( + ctx, confidence_threshold=0.2, classifier=classifier, decision_log=log, + ) + assert ctx.metadata[CONTEXT_KEY] == "dimensions" + assert tier == CAPABLE + assert classifier._client.calls == 0 # type: ignore[attr-defined] + ctx = await _ctx(seq) + assert await pick_efficient_first(ctx, confidence_threshold=0.2) == CAPABLE + + +@pytest.mark.asyncio +async def test_dimensions_routes_efficient_on_progress(): + """A deep, clean, write-heavy run scores negative → EFFICIENT for both pickers.""" + seq: list[dict] = [] + for _ in range(5): + seq.append(_msg_tool_call("Write")) + seq.append(_msg_tool_result("ok")) + seq.append({"role": "user", "content": "next"}) + ctx = await _ctx(seq) + assert await pick_capable_first(ctx, confidence_threshold=0.2) == EFFICIENT + ctx = await _ctx(seq) + assert await pick_efficient_first(ctx, confidence_threshold=0.2) == EFFICIENT @pytest.mark.asyncio @@ -106,8 +185,7 @@ async def test_no_signal_returns_default_tier(): @pytest.mark.asyncio async def test_low_confidence_consults_classifier_when_configured(): """Empty trajectory yields confidence 0.0 — picker must call the classifier.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier="efficient") assert await pick_capable_first( ctx, confidence_threshold=0.7, classifier=classifier, @@ -117,17 +195,16 @@ async def test_low_confidence_consults_classifier_when_configured(): @pytest.mark.asyncio async def test_low_confidence_falls_back_to_default_without_classifier(): - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx([{"role": "user", "content": "hi"}]) assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT @pytest.mark.asyncio async def test_classifier_fall_open_on_unknown_tier(): """A malformed classifier verdict must not override the picker default.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier=None) assert await pick_capable_first( ctx, confidence_threshold=0.7, classifier=classifier, @@ -136,45 +213,17 @@ async def test_classifier_fall_open_on_unknown_tier(): @pytest.mark.asyncio async def test_threshold_zero_skips_classifier_entirely(): - """``confidence_threshold=0`` means accept every scorer verdict, however efficient.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + """``confidence_threshold=0`` accepts every scorer verdict, however weak.""" + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier="capable") - # Empty trajectory → scorer near zero, default sign decides; classifier not consulted. await pick_capable_first(ctx, confidence_threshold=0.0, classifier=classifier) assert classifier._client.calls == 0 # type: ignore[attr-defined] -@pytest.mark.asyncio -async def test_dimensions_branch_routes_by_scorer_sign(): - """Confident scorer verdict (≥ threshold) bypasses the classifier. - - A single TodoWrite in the recent window pushes ``planning_active`` to - 1.0 (weight -0.70) — that alone clears the 0.7 confidence threshold, - so ``_pick`` takes the dimensions branch and returns EFFICIENT by sign - without consulting the classifier. - """ - log = StageRouterDecisionLog() - classifier = _stub_classifier(tier="capable") # would push CAPABLE if reached - ctx = await _ctx([ - _msg_tool_call("TodoWrite"), - _msg_tool_result("ok"), - {"role": "user", "content": "next"}, - ]) - tier = await pick_capable_first( - ctx, confidence_threshold=0.7, classifier=classifier, decision_log=log, - ) - assert ctx.metadata[CONTEXT_KEY] == "dimensions" - assert tier == EFFICIENT # negative score → EFFICIENT regardless of capable-first fallback - assert classifier._client.calls == 0 # type: ignore[attr-defined] - assert log.snapshot()["dimensions"] == 1 - - @pytest.mark.asyncio async def test_decision_log_counts_sources(): """Each decision path increments exactly one bucket in the shared log.""" log = StageRouterDecisionLog() - # override path: critical severity → CAPABLE, source=override ctx_critical = await _ctx([ _msg_tool_result("Out of memory: cannot allocate memory"), {"role": "user", "content": "retry"}, @@ -182,12 +231,10 @@ async def test_decision_log_counts_sources(): await pick_capable_first(ctx_critical, confidence_threshold=0.7, decision_log=log) assert ctx_critical.metadata[CONTEXT_KEY] == "override" - # fall_open path: low confidence, no classifier → default tier ctx_neutral = await _ctx([{"role": "user", "content": "hi"}]) await pick_capable_first(ctx_neutral, confidence_threshold=0.99, decision_log=log) assert ctx_neutral.metadata[CONTEXT_KEY] == "fall_open" - # classifier path: low confidence, classifier configured → classifier verdict classifier = _stub_classifier(tier="efficient") ctx_class = await _ctx([{"role": "user", "content": "hi"}]) await pick_capable_first( diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py deleted file mode 100644 index ed2aa6b0..00000000 --- a/tests/test_stage_router_scorer.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the stage-router dimensions + scorer.""" - -from __future__ import annotations - -import pytest - -from switchyard.lib.processors.stage_router.dimensions import ( - CodingAgentDimensions, - from_signal, -) -from switchyard.lib.processors.stage_router.scorer import DEFAULT_WEIGHTS, score -from switchyard_rust.components import DimensionCollector -from switchyard_rust.core import ChatRequest, ProxyContext - - -def _zero_dimensions() -> CodingAgentDimensions: - return CodingAgentDimensions( - severity=0.0, - no_error_streak_intensity=0.0, - write_intensity=0.0, - edit_intensity=0.0, - recent_write_intensity=0.0, - planning_active=0.0, - pure_bash_intensity=0.0, - stuck_exploring=0.0, - no_progress=0.0, - tests_passed=0.0, - ) - - -def test_zero_signal_scores_to_zero(): - result = score(_zero_dimensions()) - assert result.score == 0.0 - assert result.confidence == 0.0 - - -def test_critical_severity_pushes_toward_capable(): - dims = _zero_dimensions() - dims = CodingAgentDimensions(**{**dims.__dict__, "severity": 1.0}) - result = score(dims) - assert result.score > 0 - assert result.confidence == abs(result.score) - - -def test_tests_passed_pushes_toward_efficient(): - dims = _zero_dimensions() - dims = CodingAgentDimensions(**{**dims.__dict__, "tests_passed": 1.0}) - result = score(dims) - assert result.score < 0 - - -def test_score_is_clipped_to_unit_interval(): - """Out-of-range weighted sums must be clamped to [-1, 1].""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - # Force overflow on both sides: a 5.0-magnitude weight × 1.0 dimension - # yields raw ±5.0, which must clip to ±1.0 with confidence 1.0. - high = score(dims, weights={"severity": 5.0}) - assert high.score == 1.0 - assert high.confidence == 1.0 - low = score(dims, weights={"severity": -5.0}) - assert low.score == -1.0 - assert low.confidence == 1.0 - - -def test_custom_weights_can_invert_decision(): - """Researchers override weights via the call site, not YAML.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - default = score(dims, weights=DEFAULT_WEIGHTS) - inverted = score(dims, weights={"severity": -0.5}) - assert default.score > 0 - assert inverted.score < 0 - - -def test_contributions_sum_matches_unclipped_score(): - """When no clipping fires, ``sum(contributions) == score`` exactly.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "tests_passed": 1.0}) - result = score(dims) - expected = sum(result.contributions.values()) - assert abs(expected - result.score) < 1e-9 - - -def test_contributions_can_exceed_clipped_score(): - """When clipping fires, ``sum(contributions)`` may exceed ``|score|``.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - # Raw = 5.0 → clipped to 1.0; contributions remain unclipped. - result = score(dims, weights={"severity": 5.0}) - raw = sum(result.contributions.values()) - assert raw == 5.0 - assert result.score == 1.0 - assert raw > result.score - - -@pytest.mark.asyncio -async def test_from_signal_normalises_real_extracted_signal(): - """End-to-end: DimensionCollector → ToolResultSignal → CodingAgentDimensions.""" - collector = DimensionCollector() - ctx = ProxyContext() - request = ChatRequest.openai_chat({ - "messages": [ - {"role": "assistant", - "tool_calls": [{"function": {"name": "Write", "arguments": "{}"}}]}, - {"role": "tool", "tool_call_id": "x", "content": "ok"}, - {"role": "user", "content": "next"}, - ] - }) - await collector.process(ctx, request) - from switchyard_rust.components import get_tool_result_signal - signal = get_tool_result_signal(ctx) - assert signal is not None - dims = from_signal(signal) - assert 0.0 <= dims.severity <= 1.0 - assert 0.0 <= dims.write_intensity <= 1.0 - assert dims.write_intensity > 0 # we issued one Write call diff --git a/tests/test_tool_result_signal_collector.py b/tests/test_tool_result_signal_collector.py index 8443ca37..5c584647 100644 --- a/tests/test_tool_result_signal_collector.py +++ b/tests/test_tool_result_signal_collector.py @@ -42,7 +42,6 @@ async def test_traceback_stamps_hard_severity(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.7) - assert "traceback" in list(signal.patterns) async def test_oom_stamps_critical_severity(): @@ -65,7 +64,6 @@ async def test_clean_result_has_zero_severity(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.0) - assert list(signal.patterns) == [] async def test_no_tool_results_has_zero_severity(): @@ -175,7 +173,6 @@ async def test_anthropic_tool_result_extracted(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.7) - assert "import_error" in list(signal.patterns) # ─── OpenAI Responses format ──────────────────────────────────────────────────