From dc442bba2cb26f066ab1584220cededca261905e Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Tue, 14 Jul 2026 14:02:23 -0700 Subject: [PATCH 01/21] feat(stage_router): corroborative tanh-scored routing with windowed self-clearing signals Signed-off-by: Sabhatina Selvam --- .../processors/stage_router/decision_log.py | 3 +- .../lib/processors/stage_router/dimensions.py | 24 ++++-- .../lib/processors/stage_router/picker.py | 24 +++--- .../lib/processors/stage_router/scorer.py | 83 ++++++++++++++---- tests/test_stage_router_pickers.py | 38 +++++--- tests/test_stage_router_scorer.py | 86 ++++++++++++++----- 6 files changed, 187 insertions(+), 71 deletions(-) diff --git a/switchyard/lib/processors/stage_router/decision_log.py b/switchyard/lib/processors/stage_router/decision_log.py index 88812196..3409da42 100644 --- a/switchyard/lib/processors/stage_router/decision_log.py +++ b/switchyard/lib/processors/stage_router/decision_log.py @@ -8,7 +8,8 @@ from typing import Literal DecisionSource = Literal[ - "override", # _apply_overrides short-circuited (severity ≥ 1.0, large prompt, tests_passed) + "override", # _apply_overrides short-circuited (severity ≥ 1.0) + "tests_passed", # settled run (recent test-pass + recent write) → EFFICIENT "dimensions", # scorer confidence ≥ confidence_threshold "llm-classifier", # classifier consulted and returned a tier "fall_open", # classifier configured but returned None, OR not configured at all diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py index ce0fecbe..33a1ee64 100644 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ b/switchyard/lib/processors/stage_router/dimensions.py @@ -46,19 +46,31 @@ class CodingAgentDimensions: def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: """Project a :class:`ToolResultSignal` onto the normalised dimension space. - Note: `read_count` and `turn_depth` are still read from `signal` for the - `stuck_exploring` / `no_progress` boolean gates, but their normalised - intensities aren't exposed as separate dimensions because nothing in + Note: `turn_depth`, `recent_read_count`, `recent_write_count`, `write_count`, + and `edit_count` are read from `signal` for the `stuck_exploring` / + `no_progress` boolean gates, but their normalised intensities aren't + exposed as separate dimensions because nothing in :data:`DEFAULT_WEIGHTS` keys off them. """ total_tool_ops = signal.write_count + signal.edit_count + signal.read_count recent_tool_ops = signal.recent_write_count + signal.recent_edit_count + signal.recent_read_count + # stuck_exploring: a *recent* read-stall — spinning on reads without writing + # in the recent window. Windowed, so it drops the moment a write lands. stuck = ( signal.turn_depth >= 8 - and signal.write_count <= 1 - and signal.read_count >= 5 + and signal.recent_write_count == 0 + and signal.recent_read_count >= 2 + ) + # no_progress: a *whole-task* dead-end — deep into the run having produced + # nothing at all (no write or edit ever). Distinct from stuck_exploring (which + # only looks at the recent window), so the two are independent corroborating + # signals rather than the same condition counted twice. Cumulative by + # necessity to capture "the whole task", but self-releases on the first write. + no_progress = ( + signal.turn_depth > 30 + and signal.write_count == 0 + and signal.edit_count == 0 ) - no_progress = signal.turn_depth > 60 and signal.write_count == 0 return CodingAgentDimensions( severity=float(signal.severity), no_error_streak_intensity=_saturating(signal.no_error_streak, scale=3.0), diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index ce246129..beb69e64 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -31,10 +31,6 @@ # diverges across deployments. #: Force CAPABLE when the latest tool result hit a CRITICAL severity pattern. SEVERITY_CRITICAL: float = 1.0 -#: Force EFFICIENT when `tests_passed` AND the agent has been working long enough -#: (turn_depth) with few writes — interpreted as the run already settled. -CLEAN_TESTS_MIN_TURN_DEPTH: int = 10 -CLEAN_TESTS_MAX_WRITES: int = 1 async def pick_capable_first( @@ -93,7 +89,15 @@ async def _pick( if override is not None: return _record(ctx, decision_log, "override", override) + # Settled run: a recent test-pass backed by a recent code change (write or + # edit) is safe to run on the cheap tier — EFFICIENT for both pickers. + # Windowed, so it lapses once the run moves on. + if signal.tests_passed and (signal.recent_write_count + signal.recent_edit_count) >= 1: + return _record(ctx, decision_log, "tests_passed", EFFICIENT) + dimensions = from_signal(signal) + # Fixed weights; confidence_threshold is the corroboration dial + # (signals-to-clear = threshold / signal unit). result = score(dimensions, weights=weights) if result.confidence >= confidence_threshold: tier = CAPABLE if result.score > 0 else EFFICIENT @@ -127,15 +131,13 @@ def _record( def _apply_overrides(signal: "ToolResultSignal") -> int | None: - """Non-negotiable, signal-derived shortcuts that bypass the scorer.""" + """Non-negotiable, signal-derived shortcuts that bypass the scorer. + + A CRITICAL severity always forces CAPABLE; it outranks the settled-run + (`tests_passed`) shortcut handled in :func:`_pick`. + """ if signal.severity >= SEVERITY_CRITICAL: return CAPABLE - if ( - signal.tests_passed - and signal.turn_depth >= CLEAN_TESTS_MIN_TURN_DEPTH - and signal.write_count <= CLEAN_TESTS_MAX_WRITES - ): - return EFFICIENT return None diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py index 0c3763ad..00a470a1 100644 --- a/switchyard/lib/processors/stage_router/scorer.py +++ b/switchyard/lib/processors/stage_router/scorer.py @@ -1,29 +1,70 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Weighted linear scorer: signed score in ``[-1, +1]``, confidence = ``abs(score)``.""" +"""Weighted linear scorer, tanh-squashed to ``(-1, +1)``; confidence = ``abs(score)``.""" from __future__ import annotations +import math from collections.abc import Mapping from dataclasses import dataclass, field from switchyard.lib.processors.stage_router.dimensions import CodingAgentDimensions -#: Default linear weights. Positive ⇒ CAPABLE; negative ⇒ EFFICIENT. Calibrated so -#: a single high-impact axis lands past the 0.5 default confidence threshold. -DEFAULT_WEIGHTS: Mapping[str, float] = { - "severity": 0.80, - "stuck_exploring": 0.70, - "no_progress": 0.60, - "tests_passed": -0.80, - "planning_active": -0.70, - "write_intensity": -0.40, - "edit_intensity": -0.30, - "recent_write_intensity": -0.30, - "pure_bash_intensity": -0.30, - "no_error_streak_intensity": -0.20, -} +#: Gain applied before the tanh squash. The raw weighted sum is small (typical +#: multi-signal turns land near ±0.35), so ``tanh(gain * raw)`` spreads it across +#: most of ``(-1, +1)``. That turns the symmetric ``confidence_threshold`` into a +#: full-range dial: users pick a positive ``t`` in ``(0, 1)`` and the confident +#: fraction sweeps smoothly (e.g. t=0.1 → most turns, t=0.7 → few). Monotonic, so +#: routing sign is unchanged — only the confidence scale is spread out. +_SCORE_GAIN: float = 5.0 + +#: Max severity that reaches the scorer: `1.0` (critical) is caught by the picker +#: override, so `0.7` (hard) is the strongest error the linear scorer ever sees. +_HARD_SEVERITY: float = 0.7 +#: Fixed weight one maxed signal contributes. Weights are constant (independent of +#: threshold), so the ``confidence_threshold`` is the sole corroboration dial: +#: ``signals-to-clear = threshold / _SIGNAL_UNIT``. With unit 0.10 the threshold +#: maps to whole signal counts — 0.10 → one maxed signal clears, 0.20 → two must +#: agree, etc. (unit and threshold are redundant for routing; only their ratio +#: matters, so tuning lives entirely in the threshold). Kept well under 1 so the +#: summed score never saturates to ±1 — no single axis can peg the decision. +_SIGNAL_UNIT: float = 0.10 + +#: "Something is wrong" signals — always push toward CAPABLE (strong), both pickers. +#: severity is per-turn; the other two are windowed gates — all self-clear. +_WRONG_SIGNALS: tuple[str, ...] = ("severity", "stuck_exploring", "no_progress") +#: "Progress / settled" signals — push toward EFFICIENT (weak), both pickers. +#: capable_first / efficient_first differ only in the fall_open default. All windowed. +_PROGRESS_SIGNALS: tuple[str, ...] = ( + "recent_write_intensity", + "planning_active", + "pure_bash_intensity", + "no_error_streak_intensity", +) +#: Max value each signal reaches at the scorer; used to normalise so a maxed +#: signal contributes exactly ``_SIGNAL_UNIT``. Defaults to 1.0 for [0, 1] gates. +_MAX_VALUE: Mapping[str, float] = {"severity": _HARD_SEVERITY} + + +def _build_weights(unit: float = _SIGNAL_UNIT) -> dict[str, float]: + """Signed, fixed linear weights: wrong → CAPABLE (+), progress → EFFICIENT (-). + + A single maxed signal contributes ``unit`` (severity is normalised by its + 0.7 cap so it too lands at ``unit``). Because every weight is well under 1, + the summed score never saturates; corroboration is set downstream by the + ``confidence_threshold`` (signals-to-clear = threshold / unit). + """ + weights: dict[str, float] = {} + for name in _WRONG_SIGNALS: + weights[name] = unit / _MAX_VALUE.get(name, 1.0) + for name in _PROGRESS_SIGNALS: + weights[name] = -unit / _MAX_VALUE.get(name, 1.0) + return weights + + +#: Fixed scorer weights. Corroboration is dialed by the confidence_threshold. +DEFAULT_WEIGHTS: Mapping[str, float] = _build_weights() @dataclass(frozen=True) @@ -40,7 +81,13 @@ def score( *, weights: Mapping[str, float] = DEFAULT_WEIGHTS, ) -> ScoreResult: - """Score ``dimensions`` against ``weights``; raw sum is clipped to ``[-1, +1]``.""" + """Score ``dimensions``; raw weighted sum is tanh-squashed into ``(-1, +1)``. + + ``contributions`` are the raw per-signal weighted values (pre-squash, so they + sum to the raw score); ``score`` is ``tanh(gain * raw)``. The squash spreads + the small raw sum across the range so the ``confidence_threshold`` reads as a + symmetric ``(0, 1)`` dial. + """ contributions: dict[str, float] = {} raw = 0.0 for field_name, weight in weights.items(): @@ -48,8 +95,8 @@ def score( contribution = value * weight contributions[field_name] = contribution raw += contribution - clipped = max(-1.0, min(1.0, raw)) - return ScoreResult(score=clipped, confidence=abs(clipped), contributions=contributions) + squashed = math.tanh(_SCORE_GAIN * raw) + return ScoreResult(score=squashed, confidence=abs(squashed), contributions=contributions) __all__ = ["DEFAULT_WEIGHTS", "ScoreResult", "score"] diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py index 3fa6f3fd..f056410d 100644 --- a/tests/test_stage_router_pickers.py +++ b/tests/test_stage_router_pickers.py @@ -89,11 +89,14 @@ async def test_critical_severity_overrides_to_capable(): @pytest.mark.asyncio -async def test_tests_passed_with_edits_drops_to_efficient(): +async def test_tests_passed_with_edits_routes_to_efficient(): + """A settled run (recent test-pass + recent edit) is safe on the cheap tier.""" messages = [_msg_tool_call("Edit")] * 3 + [_msg_tool_result("all tests passed")] * 3 messages += [{"role": "user", "content": "ok continue"}] * 12 ctx = await _ctx(messages) assert await pick_capable_first(ctx, confidence_threshold=0.7) == EFFICIENT + ctx = await _ctx(messages) + assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT @pytest.mark.asyncio @@ -147,27 +150,34 @@ async def test_threshold_zero_skips_classifier_entirely(): @pytest.mark.asyncio async def test_dimensions_branch_routes_by_scorer_sign(): - """Confident scorer verdict (≥ threshold) bypasses the classifier. + """A corroborated scorer verdict (≥ threshold) bypasses the classifier. - A single TodoWrite in the recent window pushes ``planning_active`` to - 1.0 (weight -0.70) — that alone clears the 0.7 confidence threshold, - so ``_pick`` takes the dimensions branch and returns EFFICIENT by sign - without consulting the classifier. + No single signal clears threshold (weights are corroborative). A long, + write-free read spree maxes three wrong signals at once — ``stuck_exploring``, + ``no_progress`` (turn_depth > 60), and ``severity`` (the last result errors) — + which together clear the threshold, so ``_pick`` takes the dimensions branch + and routes to CAPABLE by sign without consulting the classifier. """ log = StageRouterDecisionLog() - classifier = _stub_classifier(tier="capable") # would push CAPABLE if reached - ctx = await _ctx([ - _msg_tool_call("TodoWrite"), - _msg_tool_result("ok"), - {"role": "user", "content": "next"}, - ]) + classifier = _stub_classifier(tier="efficient") # would push EFFICIENT if reached + read_spree = [] + for i in range(40): + read_spree.append(_msg_tool_call("Read")) + read_spree.append(_msg_tool_result( + "bash: foo: command not found" if i == 39 else "file contents" + )) + read_spree.append({"role": "user", "content": "next"}) + ctx = await _ctx(read_spree) tier = await pick_capable_first( - ctx, confidence_threshold=0.7, classifier=classifier, decision_log=log, + ctx, confidence_threshold=0.20, classifier=classifier, decision_log=log, ) assert ctx.metadata[CONTEXT_KEY] == "dimensions" - assert tier == EFFICIENT # negative score → EFFICIENT regardless of capable-first fallback + assert tier == CAPABLE # wrong signals → CAPABLE by sign, regardless of picker assert classifier._client.calls == 0 # type: ignore[attr-defined] assert log.snapshot()["dimensions"] == 1 + # efficient_first routes the same corroborated wrong verdict to CAPABLE too. + ctx = await _ctx(read_spree) + assert await pick_efficient_first(ctx, confidence_threshold=0.20) == CAPABLE @pytest.mark.asyncio diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py index ed2aa6b0..6a49445a 100644 --- a/tests/test_stage_router_scorer.py +++ b/tests/test_stage_router_scorer.py @@ -5,13 +5,19 @@ from __future__ import annotations +import math + import pytest from switchyard.lib.processors.stage_router.dimensions import ( CodingAgentDimensions, from_signal, ) -from switchyard.lib.processors.stage_router.scorer import DEFAULT_WEIGHTS, score +from switchyard.lib.processors.stage_router.scorer import ( + _SCORE_GAIN, + DEFAULT_WEIGHTS, + score, +) from switchyard_rust.components import DimensionCollector from switchyard_rust.core import ChatRequest, ProxyContext @@ -45,24 +51,61 @@ def test_critical_severity_pushes_toward_capable(): assert result.confidence == abs(result.score) -def test_tests_passed_pushes_toward_efficient(): +def test_tests_passed_is_not_scored(): + """tests_passed is routed to the picker's default tier, not scored here.""" dims = _zero_dimensions() dims = CodingAgentDimensions(**{**dims.__dict__, "tests_passed": 1.0}) result = score(dims) - assert result.score < 0 + assert result.score == 0.0 + +def test_progress_signal_points_efficient(): + """A progress signal scores negative (→EFFICIENT), picker-independent.""" + dims = CodingAgentDimensions( + **{**_zero_dimensions().__dict__, "recent_write_intensity": 1.0} + ) + assert score(dims).score < 0 + + +def test_wrong_signal_points_capable(): + """A wrong signal scores positive (→CAPABLE), picker-independent.""" + dims = CodingAgentDimensions( + **{**_zero_dimensions().__dict__, "stuck_exploring": 1.0} + ) + assert score(dims).score > 0 -def test_score_is_clipped_to_unit_interval(): - """Out-of-range weighted sums must be clamped to [-1, 1].""" + +def test_tanh_spread_and_monotonic_confidence(): + """Raw sum is tanh-squashed: one signal spreads into the range, more agreeing + signals raise confidence monotonically, and realistic turns never saturate. + """ + one = score( + CodingAgentDimensions(**{**_zero_dimensions().__dict__, "stuck_exploring": 1.0}) + ) + assert one.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) # tanh(gain*unit) + assert one.score > 0 # wrong signal → CAPABLE + + two = score( + CodingAgentDimensions(**{ + **_zero_dimensions().__dict__, + "recent_write_intensity": 1.0, + "pure_bash_intensity": 1.0, + }) + ) + assert two.confidence > one.confidence # more signals → more confident + assert two.score < 0 # progress → EFFICIENT + assert abs(two.score) < 1.0 # realistic turns never saturate + + +def test_extreme_weights_saturate_via_tanh(): + """A large raw sum saturates smoothly to ±1 through tanh (no hard clip).""" dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - # Force overflow on both sides: a 5.0-magnitude weight × 1.0 dimension - # yields raw ±5.0, which must clip to ±1.0 with confidence 1.0. - high = score(dims, weights={"severity": 5.0}) - assert high.score == 1.0 - assert high.confidence == 1.0 + high = score(dims, weights={"severity": 5.0}) # raw 5.0 → tanh(25) ≈ 1.0 + assert high.score == pytest.approx(1.0) + assert high.confidence == pytest.approx(1.0) low = score(dims, weights={"severity": -5.0}) - assert low.score == -1.0 - assert low.confidence == 1.0 + assert low.score == pytest.approx(-1.0) + assert low.confidence == pytest.approx(1.0) def test_custom_weights_can_invert_decision(): @@ -74,22 +117,23 @@ def test_custom_weights_can_invert_decision(): assert inverted.score < 0 -def test_contributions_sum_matches_unclipped_score(): - """When no clipping fires, ``sum(contributions) == score`` exactly.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "tests_passed": 1.0}) +def test_contributions_are_raw_presquash(): + """``contributions`` are pre-squash: they sum to raw, score = tanh(gain*raw).""" + dims = CodingAgentDimensions( + **{**_zero_dimensions().__dict__, "recent_write_intensity": 0.5} + ) result = score(dims) - expected = sum(result.contributions.values()) - assert abs(expected - result.score) < 1e-9 + raw = sum(result.contributions.values()) + assert result.score == pytest.approx(math.tanh(_SCORE_GAIN * raw)) -def test_contributions_can_exceed_clipped_score(): - """When clipping fires, ``sum(contributions)`` may exceed ``|score|``.""" +def test_contributions_exceed_squashed_score(): + """A large raw sum stays unsquashed in contributions while score saturates.""" dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - # Raw = 5.0 → clipped to 1.0; contributions remain unclipped. result = score(dims, weights={"severity": 5.0}) raw = sum(result.contributions.values()) assert raw == 5.0 - assert result.score == 1.0 + assert result.score == pytest.approx(1.0) assert raw > result.score From ad67a9c486d61ff85d715a01ad602abd1668b67d Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Tue, 14 Jul 2026 23:03:32 -0700 Subject: [PATCH 02/21] feat(stage_router): efficient_first escalates to strong on any wrong signal Signed-off-by: Sabhatina Selvam --- .../processors/stage_router/decision_log.py | 1 + .../lib/processors/stage_router/picker.py | 13 ++++++++++ tests/test_stage_router_pickers.py | 24 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/switchyard/lib/processors/stage_router/decision_log.py b/switchyard/lib/processors/stage_router/decision_log.py index 3409da42..aed0a22f 100644 --- a/switchyard/lib/processors/stage_router/decision_log.py +++ b/switchyard/lib/processors/stage_router/decision_log.py @@ -10,6 +10,7 @@ DecisionSource = Literal[ "override", # _apply_overrides short-circuited (severity ≥ 1.0) "tests_passed", # settled run (recent test-pass + recent write) → EFFICIENT + "ef_escalate", # efficient_first: any wrong signal (error/stuck/no-progress) → CAPABLE "dimensions", # scorer confidence ≥ confidence_threshold "llm-classifier", # classifier consulted and returned a tier "fall_open", # classifier configured but returned None, OR not configured at all diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index beb69e64..362a8f1b 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -96,6 +96,19 @@ async def _pick( return _record(ctx, decision_log, "tests_passed", EFFICIENT) dimensions = from_signal(signal) + # efficient_first is weak-by-default, so it escalates to CAPABLE on ANY wrong + # signal (error / stuck / no-progress) before scoring. Without this, a soft + # error — or an error diluted by a co-occurring progress signal (a write-heavy + # turn that also errored nets to ~0) — falls below the confidence bar and drops + # to the EFFICIENT default, leaving a failing turn on the weak tier. capable_first + # already defaults to CAPABLE, so it needs no such bias. + if default_tier == EFFICIENT and ( + dimensions.severity > 0.0 + or dimensions.stuck_exploring > 0.0 + or dimensions.no_progress > 0.0 + ): + return _record(ctx, decision_log, "ef_escalate", CAPABLE) + # Fixed weights; confidence_threshold is the corroboration dial # (signals-to-clear = threshold / signal unit). result = score(dimensions, weights=weights) diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py index f056410d..863bb09c 100644 --- a/tests/test_stage_router_pickers.py +++ b/tests/test_stage_router_pickers.py @@ -99,6 +99,30 @@ async def test_tests_passed_with_edits_routes_to_efficient(): assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT +@pytest.mark.asyncio +async def test_efficient_first_escalates_on_any_error(): + """efficient_first escalates to CAPABLE on any wrong signal — even a low-confidence + one that would otherwise fall through to the EFFICIENT default — so a failing turn + never stays on the weak tier. capable_first is unaffected (already CAPABLE-default).""" + log = StageRouterDecisionLog() + ctx = await _ctx([ + _msg_tool_call("Bash"), + _msg_tool_result("bash: foo: command not found"), + {"role": "user", "content": "retry"}, + ]) + # a single soft error is below the confidence bar, yet ef still escalates + assert await pick_efficient_first(ctx, confidence_threshold=0.30, decision_log=log) == CAPABLE + assert ctx.metadata[CONTEXT_KEY] == "ef_escalate" + # the bias is one-sided: capable_first keeps its own path, still CAPABLE + ctx = await _ctx([ + _msg_tool_call("Bash"), + _msg_tool_result("bash: foo: command not found"), + {"role": "user", "content": "retry"}, + ]) + assert await pick_capable_first(ctx, confidence_threshold=0.30) == CAPABLE + assert ctx.metadata[CONTEXT_KEY] != "ef_escalate" + + @pytest.mark.asyncio async def test_no_signal_returns_default_tier(): ctx = ProxyContext() # signal not stamped From 0f0091f35feb512eb543c1996b9e75fc394bbfb4 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 15 Jul 2026 23:53:47 -0700 Subject: [PATCH 03/21] feat(stage-router): window severity and no_progress over the recent window Signed-off-by: Sabhatina Selvam --- .../src/dimension_collector/tool_signals.rs | 1123 ++++++++++++++++- .../lib/processors/stage_router/dimensions.py | 14 +- 2 files changed, 1113 insertions(+), 24 deletions(-) diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index f0440be5..7a0dcb7d 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -15,22 +15,205 @@ use switchyard_protocol::{Metadata, Request, WireFormat}; /// (the `ToolResultSignal` stamped on `ProxyContext`) see a single type. pub use switchyard_libsy::{ToolSignals as ToolResultSignal, DEFAULT_RECENT_WINDOW}; -/// Adapt a format-tagged [`ChatRequest`] to a [`switchyard_protocol::Request`] -/// carrying the raw body and its wire format, which is all libsy's extractor reads. -fn to_protocol_request(request: &ChatRequest) -> Request { - let wire_format = match request.request_type() { - ChatRequestType::OpenAiChat => WireFormat::OpenAiChat, - ChatRequestType::Anthropic => WireFormat::AnthropicMessages, - ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, - }; - Request { - raw_request: Some(request.body().clone()), - metadata: Some(Metadata { - wire_format: Some(wire_format), - ..Default::default() - }), - ..Default::default() - } +const SOFT: f32 = 0.3; +const HARD: f32 = 0.7; +const CRITICAL: f32 = 1.0; + +// ─── pattern table ──────────────────────────────────────────────────────────── + +/// (name, severity, lower-cased substrings — any hit fires the pattern) +static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ + ( + "oom", + CRITICAL, + &["out of memory", "memoryerror", "cannot allocate memory"], + ), + ( + "connection_refused", + CRITICAL, + &[ + "connection refused", + "connectionrefusederror", + "econnrefused", + ], + ), + ("traceback", HARD, &["traceback (most recent call last)"]), + ( + "import_error", + HARD, + &["modulenotfounderror:", "importerror:", "no module named "], + ), + ( + "cmd_not_found", + HARD, + &["command not found", "not found\n", "/usr/bin/env: "], + ), + ("assertion", HARD, &["assertionerror"]), + ("value_error", HARD, &["valueerror:"]), + ("syntax_error", HARD, &["syntaxerror:"]), + ( + "timeout", + HARD, + &[ + "timed out", + "timeouterror", + "timeout expired", + "deadline exceeded", + ], + ), + ( + "no_such_file", + HARD, + &["filenotfounderror:", "no such file or directory"], + ), + // SOFT: plain non-zero exit without a recognisable exception traceback. + ( + "exit_nonzero", + SOFT, + &[ + "exit code 1", + "exit code 2", + "exit status 1", + "returned non-zero", + "exited with code", + ], + ), +]; + +static EDIT_TOOL_NAMES: &[&str] = &[ + "edit", + "multiedit", + "notebookedit", + "str_replace", + "str_replace_based_edit_tool", + "text_editor", +]; + +static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file"]; + +// Bash subcommand patterns. Lowercased; callers must lowercase the command +// before matching. Bucketed into write_count / edit_count alongside the +// dedicated `Write` / `Edit` tools. +static BASH_WRITE_PATTERNS: &[&str] = &[ + "cat >", + "cat >>", + "echo >", + "echo >>", + "tee ", + "printf >", + "printf >>", + "> /", + ">> /", + "<< 'eof'", + "<, + /// Consecutive clean tool results back from the most recent. `0` if the last failed. + pub no_error_streak: u32, + pub edit_count: u32, + pub write_count: u32, + /// Read-type calls (Read tool + read-like Bash). Used by the build-pit gate. + pub read_count: u32, + /// TodoWrite tool calls. Strong fail predictor for Opus; used by the + /// `capable_first` drop-to-weak gate. + pub todowrite_count: u32, + /// Edit-type calls within the last [`RECENT_WINDOW`] tool calls. + pub recent_edit_count: u32, + /// Write-type calls within the last [`RECENT_WINDOW`] tool calls. + pub recent_write_count: u32, + /// Read-type calls within the last [`RECENT_WINDOW`] tool calls. + pub recent_read_count: u32, + /// TodoWrite calls within the last [`RECENT_WINDOW`] tool calls. + pub recent_todowrite_count: u32, + /// Consecutive trailing tool calls that hit no Write/Edit/Read/Plan + /// patterns — proxy for the "stuck in non-Read Bash" build-pit loop. + pub pure_bash_streak: u32, + /// At least one of the last three tool results matched a test-pass pattern. + pub tests_passed: bool, + /// Message-count proxy for turn depth. + pub turn_depth: u32, + /// Char count of the last user message. + pub prompt_char_count: u32, } /// Extract tool-execution signals using the default `recent_*` window. @@ -43,5 +226,911 @@ pub fn extract_tool_signals_with_window( request: &ChatRequest, recent_window: usize, ) -> ToolResultSignal { - ToolResultSignal::from_request(&to_protocol_request(request), Some(recent_window)) + let body = request.body(); + let Some(obj) = body.as_object() else { + return ToolResultSignal::default(); + }; + + match request.request_type() { + ChatRequestType::Anthropic => { + let messages = obj + .get("messages") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + extract_from_messages_anthropic(messages, recent_window) + } + ChatRequestType::OpenAiResponses => { + let items = obj + .get("input") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + extract_from_input_responses(items, recent_window) + } + ChatRequestType::OpenAiChat => { + let messages = obj + .get("messages") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + extract_from_messages_openai_chat(messages, recent_window) + } + } +} + +// ─── format-specific extractors ────────────────────────────────────────────── + +fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolResultSignal { + let mut tool_texts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + let mut prompt_char_count: u32 = 0; + + for msg in messages { + let Some(obj) = msg.as_object() else { continue }; + let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); + match role { + "tool" => { + if let Some(text) = content_to_text(obj.get("content")) { + tool_texts.push(text); + } + } + "assistant" => { + if let Some(tc_list) = obj.get("tool_calls").and_then(Value::as_array) { + for tc in tc_list { + let Some(fn_obj) = tc + .as_object() + .and_then(|t| t.get("function")) + .and_then(|f| f.as_object()) + else { + continue; + }; + let Some(name) = fn_obj.get("name").and_then(Value::as_str) else { + continue; + }; + // OpenAI Chat encodes `arguments` as a JSON string. + let command = fn_obj + .get("arguments") + .and_then(Value::as_str) + .and_then(|s| serde_json::from_str::(s).ok()) + .and_then(|v| { + v.get("command") + .and_then(Value::as_str) + .map(|s| s.to_lowercase()) + }); + tool_calls.push(ObservedToolCall { + name: name.to_string(), + command, + }); + } + } + } + "user" => { + prompt_char_count = user_message_char_count(obj.get("content")); + } + _ => {} + } + } + + build_signal( + tool_texts, + tool_calls, + messages.len() as u32, + prompt_char_count, + recent_window, + ) +} + +fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolResultSignal { + let mut tool_texts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + let mut prompt_char_count: u32 = 0; + + for msg in messages { + let Some(obj) = msg.as_object() else { continue }; + let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); + let content = obj.get("content"); + + match role { + "user" => { + if let Some(Value::Array(blocks)) = content { + let mut saw_text = false; + for block in blocks { + let Some(b) = block.as_object() else { continue }; + match b.get("type").and_then(Value::as_str) { + Some("tool_result") => { + if let Some(text) = content_to_text(b.get("content")) { + tool_texts.push(text); + } + } + Some("text") => { + if let Some(t) = b.get("text").and_then(Value::as_str) { + prompt_char_count = t.len() as u32; + saw_text = true; + } + } + _ => {} + } + } + if !saw_text { + if let Some(text_val) = content { + prompt_char_count = user_message_char_count(Some(text_val)); + } + } + } else { + prompt_char_count = user_message_char_count(content); + } + } + "assistant" => { + if let Some(Value::Array(blocks)) = content { + for block in blocks { + let Some(b) = block.as_object() else { continue }; + if b.get("type").and_then(Value::as_str) == Some("tool_use") { + let Some(name) = b.get("name").and_then(Value::as_str) else { + continue; + }; + // Anthropic delivers `input` as a parsed object. + let command = b + .get("input") + .and_then(Value::as_object) + .and_then(|i| i.get("command")) + .and_then(Value::as_str) + .map(|s| s.to_lowercase()); + tool_calls.push(ObservedToolCall { + name: name.to_string(), + command, + }); + } + } + } + } + _ => {} + } + } + + build_signal( + tool_texts, + tool_calls, + messages.len() as u32, + prompt_char_count, + recent_window, + ) +} + +fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolResultSignal { + let mut tool_texts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + let mut prompt_char_count: u32 = 0; + + for item in items { + let Some(obj) = item.as_object() else { + continue; + }; + let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); + let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); + + match item_type { + "function_call_output" => { + if let Some(output) = obj.get("output").and_then(Value::as_str) { + tool_texts.push(output.to_string()); + } + } + "function_call" => { + let Some(name) = obj.get("name").and_then(Value::as_str) else { + continue; + }; + let command = obj + .get("arguments") + .and_then(Value::as_str) + .and_then(|s| serde_json::from_str::(s).ok()) + .and_then(|v| { + v.get("command") + .and_then(Value::as_str) + .map(|s| s.to_lowercase()) + }); + tool_calls.push(ObservedToolCall { + name: name.to_string(), + command, + }); + } + _ => { + if role == "user" { + prompt_char_count = user_message_char_count(obj.get("content")); + } + } + } + } + + build_signal( + tool_texts, + tool_calls, + items.len() as u32, + prompt_char_count, + recent_window, + ) +} + +// ─── aggregation ───────────────────────────────────────────────────────────── + +fn build_signal( + tool_texts: Vec, + tool_calls: Vec, + turn_depth: u32, + prompt_char_count: u32, + recent_window: usize, +) -> ToolResultSignal { + // Windowed severity: take the MAX severity across the last `recent_window` tool + // results rather than only the last one. An error's severity then persists for + // the recent window and decays out of it — parallel to the windowed `recent_*` + // counts and `stuck_exploring` — so a fix written a couple of turns after an error + // still routes on the error signal instead of the router flapping straight back to + // the weak tier. `patterns` carries the labels of the highest-severity result. + let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); + let mut severity = 0.0f32; + let mut patterns: Vec = Vec::new(); + for text in &tool_texts[sev_start..] { + let (sev, pats) = classify_text(text); + if sev > severity { + severity = sev; + patterns = pats; + } + } + + let no_error_streak = compute_no_error_streak(&tool_texts); + + // Single pass: cumulative + sliding-window counters together. Also tracks + // the trailing pure-bash streak (consecutive `Other`-category calls back + // from the end) — the build-pit proxy. + let recent_start = tool_calls.len().saturating_sub(recent_window); + let mut write_count = 0u32; + let mut edit_count = 0u32; + let mut read_count = 0u32; + let mut todowrite_count = 0u32; + let mut recent_write_count = 0u32; + let mut recent_edit_count = 0u32; + let mut recent_read_count = 0u32; + let mut recent_todowrite_count = 0u32; + let mut pure_bash_streak = 0u32; + let mut streak_open = true; + for (i, tc) in tool_calls.iter().enumerate().rev() { + let cat = classify_tool_call(&tc.name, tc.command.as_deref()); + if streak_open { + if matches!(cat, ToolCategory::Other) { + pure_bash_streak += 1; + } else { + streak_open = false; + } + } + match cat { + ToolCategory::Write => { + write_count += 1; + if i >= recent_start { + recent_write_count += 1; + } + } + ToolCategory::Edit => { + edit_count += 1; + if i >= recent_start { + recent_edit_count += 1; + } + } + ToolCategory::Read => { + read_count += 1; + if i >= recent_start { + recent_read_count += 1; + } + } + ToolCategory::Plan => { + todowrite_count += 1; + if i >= recent_start { + recent_todowrite_count += 1; + } + } + ToolCategory::Other => {} + } + } + + let tests_passed = detect_tests_passed(&tool_texts); + + ToolResultSignal { + severity, + patterns, + no_error_streak, + edit_count, + write_count, + read_count, + todowrite_count, + recent_edit_count, + recent_write_count, + recent_read_count, + recent_todowrite_count, + pure_bash_streak, + tests_passed, + turn_depth, + prompt_char_count, + } +} + +// ─── pure helpers ───────────────────────────────────────────────────────────── + +/// Normalise a JSON tool-result content value to a plain string. +fn content_to_text(content: Option<&Value>) -> Option { + match content? { + Value::String(s) => Some(s.clone()), + Value::Array(blocks) => { + let parts: Vec<&str> = blocks + .iter() + .filter_map(|b| { + b.as_object() + .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) + .and_then(|o| o.get("text")) + .and_then(Value::as_str) + }) + .collect(); + if parts.is_empty() { + None + } else { + Some(parts.join("\n")) + } + } + _ => None, + } +} + +/// Character count of a user-message content value. +fn user_message_char_count(content: Option<&Value>) -> u32 { + match content { + Some(Value::String(s)) => s.len() as u32, + Some(Value::Array(blocks)) => blocks + .iter() + .filter_map(|b| { + b.as_object() + .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) + .and_then(|o| o.get("text")) + .and_then(Value::as_str) + }) + .map(|s| s.len() as u32) + .sum(), + _ => 0, + } +} + +/// Match `text` against the error pattern table. +/// +/// Returns `(max_severity, matched_pattern_names)`. +pub(crate) fn classify_text(text: &str) -> (f32, Vec) { + let lower = text.to_lowercase(); + let mut patterns = Vec::new(); + let mut severity: f32 = 0.0; + for (name, sev, substrings) in ERROR_PATTERNS { + if substrings.iter().any(|sub| lower.contains(sub)) { + patterns.push(name.to_string()); + severity = severity.max(*sev); + } + } + (severity, patterns) +} + +fn compute_no_error_streak(tool_texts: &[String]) -> u32 { + let mut streak = 0u32; + for text in tool_texts.iter().rev() { + let (sev, _) = classify_text(text); + if sev > 0.0 { + break; + } + streak += 1; + } + streak +} + +fn detect_tests_passed(tool_texts: &[String]) -> bool { + let recent = if tool_texts.len() > 3 { + &tool_texts[tool_texts.len() - 3..] + } else { + tool_texts + }; + recent.iter().any(|text| { + let lower = text.to_lowercase(); + TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) + && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) + && !has_nonzero_failure_count(&lower) + }) +} + +// True iff `lower` contains a `NUMERIC_FAILURE_KEYWORDS` token preceded +// (modulo whitespace) by a nonzero integer. The "modulo whitespace" lets +// "1 failed", "1\nfailed", and "1 failed" all trip; the nonzero guard +// keeps cargo's "0 failed" / go's "0 errors" / pytest's "0 errors in" +// summaries from being misread as failures on a clean run. +fn has_nonzero_failure_count(lower: &str) -> bool { + for kw in NUMERIC_FAILURE_KEYWORDS { + let mut cursor = 0usize; + while let Some(rel) = lower[cursor..].find(kw) { + let kw_start = cursor + rel; + let kw_end = kw_start + kw.len(); + // Word boundary AFTER the keyword — "errors" mid-word (e.g. + // "errored") shouldn't count as a failure-count site. + let boundary_after = lower[kw_end..] + .chars() + .next() + .is_none_or(|c| !c.is_ascii_alphanumeric()); + if boundary_after { + let prefix = &lower[..kw_start]; + let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace()); + let digits_rev: String = trimmed + .chars() + .rev() + .take_while(|c| c.is_ascii_digit()) + .collect(); + if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') { + return true; + } + } + cursor = kw_start + kw.len(); + } + } + false +} + +// ─── tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn clean_text_has_zero_severity() { + let (sev, patterns) = classify_text("everything went fine"); + assert_eq!(sev, 0.0); + assert!(patterns.is_empty()); + } + + #[test] + fn traceback_is_hard() { + let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError"); + assert_eq!(sev, HARD); + assert!(patterns.contains(&"traceback".to_string())); + } + + #[test] + fn oom_is_critical() { + let (sev, _) = classify_text("Out of memory: kill process 1234"); + assert_eq!(sev, CRITICAL); + } + + #[test] + fn severity_is_max_across_patterns() { + // exit_nonzero (SOFT) + traceback (HARD) → HARD. + let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):"); + assert_eq!(sev, HARD); + } + + #[test] + fn no_error_streak_all_clean() { + let texts = vec!["ok".to_string(), "all good".to_string()]; + assert_eq!(compute_no_error_streak(&texts), 2); + } + + #[test] + fn no_error_streak_stops_at_error() { + let texts = vec![ + "Traceback (most recent call last):".to_string(), + "ok".to_string(), + "ok".to_string(), + ]; + assert_eq!(compute_no_error_streak(&texts), 2); + } + + #[test] + fn tests_passed_detects_pytest_output() { + assert!(detect_tests_passed(&[ + "====== 5 passed in 0.12s ======".to_string() + ])); + } + + #[test] + fn tests_passed_ignores_partial_failures() { + assert!(!detect_tests_passed(&[ + "2 failed, 5 passed in 0.56s".to_string() + ])); + } + + #[test] + fn severity_is_windowed_over_recent_results() { + // An error two results back, then two clean results. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "tool", "tool_call_id": "1", + "content": "Traceback (most recent call last):\n ValueError"}, + {"role": "tool", "tool_call_id": "2", "content": "ok"}, + {"role": "tool", "tool_call_id": "3", "content": "ok"}, + ] + })); + // window covers the error → severity persists (max over the window) + assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD); + // window of 1 sees only the last (clean) result → severity has decayed out + assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0); + } + + #[test] + fn extract_openai_chat_tool_results() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "user", "content": "do something"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, + {"role": "tool", "tool_call_id": "1", "content": "Traceback (most recent call last):\n ValueError"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.severity, HARD); + assert!(sig.patterns.contains(&"traceback".to_string())); + assert_eq!(sig.edit_count, 1); + assert_eq!(sig.turn_depth, 3); + } + + #[test] + fn extract_anthropic_tool_results() { + let request = ChatRequest::anthropic(json!({ + "messages": [ + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "1", + "content": "Traceback (most recent call last):\n ValueError"} + ]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.severity, HARD); + } + + #[test] + fn extract_responses_api_tool_results() { + let request = ChatRequest::openai_responses(json!({ + "input": [ + {"type": "function_call", "name": "Write"}, + {"type": "function_call_output", "call_id": "1", + "output": "file written successfully"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.severity, 0.0); + assert_eq!(sig.write_count, 1); + } + + #[test] + fn recent_window_counts_only_last_three_tool_calls() { + // 5 writes + 1 edit at the end → recent window of 3 should see + // 1 edit + 2 writes (not all 5 writes). + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, + {"role": "tool", "content": "ok"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.write_count, 5); + assert_eq!(sig.edit_count, 1); + assert_eq!(sig.recent_write_count, 2); + assert_eq!(sig.recent_edit_count, 1); + } + + #[test] + fn recent_window_size_is_caller_overridable() { + // Same six tool calls (1 edit at the end, 5 writes before). + // With recent_window=3 → recent_writes=2, recent_edits=1. + // With recent_window=6 → recent_writes=5, recent_edits=1 (all calls). + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, + {"role": "tool", "content": "ok"}, + ] + })); + let narrow = extract_tool_signals_with_window(&request, 3); + assert_eq!(narrow.recent_write_count, 2); + assert_eq!(narrow.recent_edit_count, 1); + + let wide = extract_tool_signals_with_window(&request, 6); + assert_eq!(wide.recent_write_count, 5); + assert_eq!(wide.recent_edit_count, 1); + } + + #[test] + fn bash_heredoc_counts_as_write() { + // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{ + "function": { + "name": "Bash", + "arguments": "{\"command\": \"cat > /tmp/test.py <<'EOF'\\nprint(1)\\nEOF\"}" + } + }]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!( + sig.write_count, 1, + "Bash heredoc should bucket into write_count" + ); + assert_eq!(sig.edit_count, 0); + } + + #[test] + fn bash_sed_inplace_counts_as_edit() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{ + "function": { + "name": "Bash", + "arguments": "{\"command\": \"sed -i 's/foo/bar/g' /app/file.py\"}" + } + }]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!( + sig.edit_count, 1, + "Bash sed -i should bucket into edit_count" + ); + assert_eq!(sig.write_count, 0); + } + + #[test] + fn bash_non_mutating_does_not_count() { + // ls, cat, grep — should not increment either counter. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{ + "function": {"name": "Bash", "arguments": "{\"command\": \"ls -la /app\"}"} + }]}, + {"role": "assistant", "tool_calls": [{ + "function": {"name": "Bash", "arguments": "{\"command\": \"cat /app/main.py\"}"} + }]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.write_count, 0); + assert_eq!(sig.edit_count, 0); + } + + #[test] + fn tests_passed_detects_pytest_with_failure_block() { + // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. + assert!(!detect_tests_passed(&[ + "2 failed, 5 passed in 0.56s".to_string() + ])); + } + + #[test] + fn tests_passed_accepts_cargo_clean_summary() { + // Cargo's clean-run summary contains "0 failed" — must not trip the + // failure list (regression: previously substring-matched "failed"). + assert!(detect_tests_passed(&[ + "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string() + ])); + } + + #[test] + fn tests_passed_rejects_cargo_real_failure() { + // Cargo's actual-failure summary: nonzero count before "failed". + assert!(!detect_tests_passed(&[ + "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string() + ])); + } + + #[test] + fn tests_passed_accepts_go_clean_summary() { + // Go test's clean-run "0 errors" must not trip (regression). + assert!(detect_tests_passed(&[ + "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string() + ])); + } + + #[test] + fn tests_passed_accepts_pytest_zero_errors() { + // Pytest long-form: "0 errors in 0.3s" on a clean run. + assert!(detect_tests_passed(&[ + "5 passed, 0 errors in 0.30s".to_string() + ])); + } + + #[test] + fn tests_passed_detects_diy_checkmark() { + assert!(detect_tests_passed(&["✓ all checks passed".to_string()])); + } + + #[test] + fn anthropic_bash_heredoc_extracts_command() { + // Anthropic format: tool_use.input is an object, not a JSON string. + let request = ChatRequest::anthropic(json!({ + "messages": [ + {"role": "assistant", "content": [ + {"type": "tool_use", "name": "Bash", + "input": {"command": "cat > /tmp/foo.txt << 'EOF'\nhi\nEOF"}} + ]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!( + sig.write_count, 1, + "Anthropic Bash heredoc must also be detected" + ); + } + + #[test] + fn recent_window_falls_back_to_full_history_when_short() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.recent_write_count, 1); + assert_eq!(sig.recent_edit_count, 0); + } + + #[test] + fn clean_tool_result_has_zero_severity_and_non_empty_streak() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "tool", "tool_call_id": "1", "content": "output ok"}, + {"role": "tool", "tool_call_id": "2", "content": "another ok"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.severity, 0.0); + assert_eq!(sig.no_error_streak, 2); + } + + // ─── asymmetric-signal extensions ──────────────────────────────────── + + #[test] + fn todowrite_classifies_as_plan() { + assert_eq!(classify_tool_call("TodoWrite", None), ToolCategory::Plan); + assert_eq!(classify_tool_call("todo_write", None), ToolCategory::Plan); + } + + #[test] + fn codex_update_plan_classifies_as_plan() { + assert_eq!(classify_tool_call("update_plan", None), ToolCategory::Plan); + } + + #[test] + fn codex_shell_command_runs_bash_pattern_match() { + // shell_command + heredoc -> Write. + assert_eq!( + classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")), + ToolCategory::Write, + ); + // shell_command + read-like inspection -> Read. + assert_eq!( + classify_tool_call("shell_command", Some("ls /app")), + ToolCategory::Read, + ); + // shell_command without matching patterns -> Other. + assert_eq!( + classify_tool_call("shell_command", Some("./run_tests.sh")), + ToolCategory::Other, + ); + } + + #[test] + fn read_tool_classifies_as_read() { + assert_eq!(classify_tool_call("Read", None), ToolCategory::Read); + assert_eq!(classify_tool_call("View", None), ToolCategory::Read); + } + + #[test] + fn bash_read_patterns_classify_as_read() { + let cases = [ + "cat /etc/passwd", + "grep foo bar.txt", + "ls /app", + "find . -name '*.py'", + ]; + for cmd in cases { + assert_eq!( + classify_tool_call("Bash", Some(cmd)), + ToolCategory::Read, + "expected Read for {cmd}" + ); + } + } + + #[test] + fn bash_write_precedence_over_read() { + // `cat /file > out` contains both `cat /` (read) and ` > ` (write); + // write redirection must win. + assert_eq!( + classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")), + ToolCategory::Write, + ); + } + + #[test] + fn pure_bash_streak_counts_trailing_other() { + // 5 trailing non-classified Bash calls → streak == 5. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"make\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"./configure\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"make install\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"./run.sh\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"./test\"}"}}]}, + {"role": "tool", "content": "ok"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.pure_bash_streak, 5); + assert_eq!(sig.write_count, 0); + assert_eq!(sig.read_count, 0); + } + + #[test] + fn pure_bash_streak_resets_on_write() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"make\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, + {"role": "tool", "content": "ok"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.pure_bash_streak, 0); + assert_eq!(sig.write_count, 1); + } + + #[test] + fn recent_window_tracks_todowrite_and_read() { + // Final 3 tool calls: TodoWrite, Read, TodoWrite. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", + "arguments": "{\"command\": \"make\"}"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Read"}}]}, + {"role": "tool", "content": "ok"}, + {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, + {"role": "tool", "content": "ok"}, + ] + })); + let sig = extract_tool_signals(&request); + assert_eq!(sig.todowrite_count, 2); + assert_eq!(sig.recent_todowrite_count, 2); + assert_eq!(sig.read_count, 1); + assert_eq!(sig.recent_read_count, 1); + } } diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py index 33a1ee64..00679a37 100644 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ b/switchyard/lib/processors/stage_router/dimensions.py @@ -61,15 +61,15 @@ def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: and signal.recent_write_count == 0 and signal.recent_read_count >= 2 ) - # no_progress: a *whole-task* dead-end — deep into the run having produced - # nothing at all (no write or edit ever). Distinct from stuck_exploring (which - # only looks at the recent window), so the two are independent corroborating - # signals rather than the same condition counted twice. Cumulative by - # necessity to capture "the whole task", but self-releases on the first write. + # no_progress: a *windowed* dead-end — deep into the run (turn_depth > 30) with no + # write or edit in the recent window. Windowed like every other signal, so it clears + # once the agent produces something recent and re-fires if it stalls again. The + # deeper turn_depth gate keeps it distinct from stuck_exploring (turn_depth >= 8 and + # requires recent reads), so the two still corroborate rather than duplicate. no_progress = ( signal.turn_depth > 30 - and signal.write_count == 0 - and signal.edit_count == 0 + and signal.recent_write_count == 0 + and signal.recent_edit_count == 0 ) return CodingAgentDimensions( severity=float(signal.severity), From d132e2a7cb02c180bcebec0094035e5370dfdb16 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 17 Jul 2026 11:48:38 -0700 Subject: [PATCH 04/21] feat(dimension-collector): add trace-mined "file does not exist" error pattern Signed-off-by: Sabhatina Selvam --- .../src/dimension_collector/tool_signals.rs | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index 7a0dcb7d..adc8ae63 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -64,7 +64,14 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ ( "no_such_file", HARD, - &["filenotfounderror:", "no such file or directory"], + &[ + "filenotfounderror:", + "no such file or directory", + // Claude Code Read-tool miss. Anchored as "file does not exist" (not a + // bare "does not exist", which fires on `ls` output and prose) — trace- + // mined across 1006 local trajectories at 22 true / 2 false positives. + "file does not exist", + ], ), // SOFT: plain non-zero exit without a recognisable exception traceback. ( @@ -706,6 +713,23 @@ mod tests { assert_eq!(sev, HARD); } + #[test] + fn file_does_not_exist_is_hard() { + // Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives). + let (sev, patterns) = + classify_text("Error: File does not exist. Note: current working directory is /app."); + assert_eq!(sev, HARD); + assert!(patterns.contains(&"no_such_file".to_string())); + } + + #[test] + fn bare_does_not_exist_stays_clean() { + // Precision guard: only the anchored "file does not exist" fires, so a bare + // "does not exist" in prose or directory output must not trip a false error. + let (sev, _) = classify_text("The directory does not exist yet, creating it now."); + assert_eq!(sev, 0.0); + } + #[test] fn no_error_streak_all_clean() { let texts = vec!["ok".to_string(), "all good".to_string()]; From e5348de2a7b5a463173842ae208b8bc20f62530e Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 17 Jul 2026 12:07:45 -0700 Subject: [PATCH 05/21] refactor(stage-router): two-axis scorer with exclusive spinning/exploring signals; window=5 Signed-off-by: Sabhatina Selvam --- .../src/dimension_collector/tool_signals.rs | 66 ++++---- .../processors/stage_router/decision_log.py | 7 +- .../lib/processors/stage_router/dimensions.py | 104 +++++++------ .../lib/processors/stage_router/picker.py | 32 ++-- .../lib/processors/stage_router/scorer.py | 85 ++++++----- tests/test_stage_router_pickers.py | 142 +++++++++--------- tests/test_stage_router_scorer.py | 140 +++++++---------- 7 files changed, 280 insertions(+), 296 deletions(-) diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index adc8ae63..9acf8393 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -138,8 +138,7 @@ static BASH_READ_PATTERNS: &[&str] = &[ static READ_TOOL_NAMES: &[&str] = &["read", "view"]; -// Planning / scratchpad tool calls. Used by Opus as a struggle indicator -// in the capable-first picker direction. +// Planning / scratchpad tool calls — investigative (non-producing) activity. // `update_plan` is codex's equivalent of `todowrite`. static PLAN_TOOL_NAMES: &[&str] = &["todowrite", "todo_write", "todo", "update_plan"]; @@ -174,13 +173,13 @@ static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "er // "0 errors" summaries on a clean run are not misread as failures. static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"]; -/// Default sliding-window size for `recent_*` counts. +/// Default sliding-window size for `recent_*` counts and windowed severity. /// -/// Calibrated against agent trajectories where a 3-call horizon is enough -/// to capture the "what is the agent doing right now" signal without -/// over-smoothing short tasks. Override per-extractor by calling -/// [`extract_tool_signals_with_window`] directly. -pub const DEFAULT_RECENT_WINDOW: usize = 3; +/// A 5-call horizon captures "what is the agent doing right now" while keeping +/// signals sticky — an error or stall persists a few recovery turns instead of +/// flickering off the moment one clean result lands. Override per-extractor by +/// calling [`extract_tool_signals_with_window`] directly. +pub const DEFAULT_RECENT_WINDOW: usize = 5; // ─── output type ───────────────────────────────────────────────────────────── @@ -201,8 +200,8 @@ pub struct ToolResultSignal { pub write_count: u32, /// Read-type calls (Read tool + read-like Bash). Used by the build-pit gate. pub read_count: u32, - /// TodoWrite tool calls. Strong fail predictor for Opus; used by the - /// `capable_first` drop-to-weak gate. + /// TodoWrite / planning tool calls. Investigative (non-producing) activity — + /// recent todowrites distinguish `exploring` from `spinning` in the scorer. pub todowrite_count: u32, /// Edit-type calls within the last [`RECENT_WINDOW`] tool calls. pub recent_edit_count: u32, @@ -212,12 +211,14 @@ pub struct ToolResultSignal { pub recent_read_count: u32, /// TodoWrite calls within the last [`RECENT_WINDOW`] tool calls. pub recent_todowrite_count: u32, - /// Consecutive trailing tool calls that hit no Write/Edit/Read/Plan - /// patterns — proxy for the "stuck in non-Read Bash" build-pit loop. + /// Consecutive trailing tool calls in the `Other` category (no Write/Edit/Read/ + /// Plan match). Surfaced in the classifier state summary; not scored directly. pub pure_bash_streak: u32, /// At least one of the last three tool results matched a test-pass pattern. pub tests_passed: bool, - /// Message-count proxy for turn depth. + /// Message-count proxy for turn depth. Wire-format dependent (Anthropic batches + /// tool results into fewer messages than OpenAI-chat), so gates keyed on it are + /// approximate across request origins. pub turn_depth: u32, /// Char count of the last user message. pub prompt_char_count: u32, @@ -537,7 +538,7 @@ fn build_signal( } } - let tests_passed = detect_tests_passed(&tool_texts); + let tests_passed = detect_tests_passed(&tool_texts, recent_window); ToolResultSignal { severity, @@ -630,13 +631,9 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 { streak } -fn detect_tests_passed(tool_texts: &[String]) -> bool { - let recent = if tool_texts.len() > 3 { - &tool_texts[tool_texts.len() - 3..] - } else { - tool_texts - }; - recent.iter().any(|text| { +fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool { + let start = tool_texts.len().saturating_sub(recent_window.max(1)); + tool_texts[start..].iter().any(|text| { let lower = text.to_lowercase(); TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) @@ -750,14 +747,14 @@ mod tests { fn tests_passed_detects_pytest_output() { assert!(detect_tests_passed(&[ "====== 5 passed in 0.12s ======".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] fn tests_passed_ignores_partial_failures() { assert!(!detect_tests_passed(&[ "2 failed, 5 passed in 0.56s".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] @@ -822,9 +819,9 @@ mod tests { } #[test] - fn recent_window_counts_only_last_three_tool_calls() { - // 5 writes + 1 edit at the end → recent window of 3 should see - // 1 edit + 2 writes (not all 5 writes). + fn recent_window_counts_only_last_default_window_tool_calls() { + // 5 writes + 1 edit at the end → the default window (5) should see + // the last 5 calls: 1 edit + 4 writes (not all 6 calls). let request = ChatRequest::openai_chat(json!({ "messages": [ {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, @@ -844,7 +841,7 @@ mod tests { let sig = extract_tool_signals(&request); assert_eq!(sig.write_count, 5); assert_eq!(sig.edit_count, 1); - assert_eq!(sig.recent_write_count, 2); + assert_eq!(sig.recent_write_count, 4); assert_eq!(sig.recent_edit_count, 1); } @@ -942,7 +939,7 @@ mod tests { // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. assert!(!detect_tests_passed(&[ "2 failed, 5 passed in 0.56s".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] @@ -951,7 +948,7 @@ mod tests { // failure list (regression: previously substring-matched "failed"). assert!(detect_tests_passed(&[ "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] @@ -959,7 +956,7 @@ mod tests { // Cargo's actual-failure summary: nonzero count before "failed". assert!(!detect_tests_passed(&[ "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] @@ -967,7 +964,7 @@ mod tests { // Go test's clean-run "0 errors" must not trip (regression). assert!(detect_tests_passed(&[ "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] @@ -975,12 +972,15 @@ mod tests { // Pytest long-form: "0 errors in 0.3s" on a clean run. assert!(detect_tests_passed(&[ "5 passed, 0 errors in 0.30s".to_string() - ])); + ], DEFAULT_RECENT_WINDOW)); } #[test] fn tests_passed_detects_diy_checkmark() { - assert!(detect_tests_passed(&["✓ all checks passed".to_string()])); + assert!(detect_tests_passed( + &["✓ all checks passed".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] diff --git a/switchyard/lib/processors/stage_router/decision_log.py b/switchyard/lib/processors/stage_router/decision_log.py index aed0a22f..97077b31 100644 --- a/switchyard/lib/processors/stage_router/decision_log.py +++ b/switchyard/lib/processors/stage_router/decision_log.py @@ -8,10 +8,9 @@ from typing import Literal DecisionSource = Literal[ - "override", # _apply_overrides short-circuited (severity ≥ 1.0) - "tests_passed", # settled run (recent test-pass + recent write) → EFFICIENT - "ef_escalate", # efficient_first: any wrong signal (error/stuck/no-progress) → CAPABLE - "dimensions", # scorer confidence ≥ confidence_threshold + "override", # _apply_overrides short-circuited (severity ≥ 1.0, CRITICAL) + "tests_passed", # settled run (recent test-pass + recent write, no error) → EFFICIENT + "dimensions", # scorer confidence ≥ confidence_threshold → routed by sign "llm-classifier", # classifier consulted and returned a tier "fall_open", # classifier configured but returned None, OR not configured at all "no_signal", # ToolResultSignal not present (first turn) diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py index 00679a37..677a6718 100644 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ b/switchyard/lib/processors/stage_router/dimensions.py @@ -1,7 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Scorer-ready view of :class:`ToolResultSignal` — all fields normalised to ``[0, 1]``.""" +"""Scorer-ready view of :class:`ToolResultSignal` — two axes, five signals. + +The stage_router scorer models a coding turn on two independent axes: + +* **error** — did the recent tool results error? ``severity`` (bad pole) vs + ``no_error_streak_intensity`` (good pole). +* **production** — is the agent producing code? ``spinning`` / ``exploring`` + (bad poles) vs ``recent_production_intensity`` (good pole). + +``spinning`` and ``exploring`` are mutually exclusive, split by whether the +agent is doing *any* investigative work (reads / plans) in the recent window: +``spinning`` = not even looking, ``exploring`` = looking but not building. + +Every field here is consumed — ``severity`` by the picker override, the rest by +the scorer weights. All counts are over the recent window; see the notes on +:func:`from_signal` for the two windowing caveats. +""" from __future__ import annotations @@ -12,12 +28,13 @@ if TYPE_CHECKING: from switchyard_rust.components import ToolResultSignal - -_PURE_BASH_NORM: float = 8.0 +#: Turn-depth below which stall signals stay quiet — early no-write turns are +#: normal exploration, not a stall. +_STALL_MIN_TURN_DEPTH: int = 8 def _saturating(x: float, scale: float) -> float: - """Map non-negative counts to ``[0, 1]``; ``scale`` is the half-saturation point.""" + """Map non-negative counts to ``[0, 1)``; ``scale`` is the half-saturation point.""" if x <= 0: return 0.0 return 1.0 - math.exp(-x / scale) @@ -31,59 +48,54 @@ def _ratio(numerator: int, denominator: int) -> float: class CodingAgentDimensions: """Normalised, scorer-ready view of a single :class:`ToolResultSignal`.""" + #: Windowed max error severity in ``[0, 1]`` (used by the picker override; the + #: scorer weights it capped at HARD since CRITICAL short-circuits upstream). severity: float + #: 1.0 when the recent window has no reads, plans, writes, or edits — the agent + #: is only cycling non-inspecting commands (a struggle signal → strong). + spinning: float + #: 1.0 when the recent window has reads/plans but no writes or edits — the agent + #: is investigating without converging (a neutral-ish signal → strong, half weight). + exploring: float + #: Fraction of recent tool ops that produced code (writes + edits) → weak. + recent_production_intensity: float + #: Saturating count of consecutive clean recent results → weak. no_error_streak_intensity: float - write_intensity: float - edit_intensity: float - recent_write_intensity: float - planning_active: float - pure_bash_intensity: float - stuck_exploring: float - no_progress: float - tests_passed: float def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: - """Project a :class:`ToolResultSignal` onto the normalised dimension space. + """Project a :class:`ToolResultSignal` onto the two-axis dimension space. + + Windowing notes (both inherent to routing on the normalised request): - Note: `turn_depth`, `recent_read_count`, `recent_write_count`, `write_count`, - and `edit_count` are read from `signal` for the `stuck_exploring` / - `no_progress` boolean gates, but their normalised intensities aren't - exposed as separate dimensions because nothing in - :data:`DEFAULT_WEIGHTS` keys off them. + * ``turn_depth`` is a raw *message* count, so its scale varies by wire format + (Anthropic batches tool results into fewer messages than OpenAI-chat). The + ``_STALL_MIN_TURN_DEPTH`` gate is therefore approximate across origins. + * ``severity`` is windowed over the last N tool *results* while the ``recent_*`` + counts are over the last N tool *calls* — usually the same turns, but a + trailing call without a result yet can offset them by one. """ - total_tool_ops = signal.write_count + signal.edit_count + signal.read_count - recent_tool_ops = signal.recent_write_count + signal.recent_edit_count + signal.recent_read_count - # stuck_exploring: a *recent* read-stall — spinning on reads without writing - # in the recent window. Windowed, so it drops the moment a write lands. - stuck = ( - signal.turn_depth >= 8 - and signal.recent_write_count == 0 - and signal.recent_read_count >= 2 - ) - # no_progress: a *windowed* dead-end — deep into the run (turn_depth > 30) with no - # write or edit in the recent window. Windowed like every other signal, so it clears - # once the agent produces something recent and re-fires if it stalls again. The - # deeper turn_depth gate keeps it distinct from stuck_exploring (turn_depth >= 8 and - # requires recent reads), so the two still corroborate rather than duplicate. - no_progress = ( - signal.turn_depth > 30 - and signal.recent_write_count == 0 - and signal.recent_edit_count == 0 + recent_ops = ( + signal.recent_write_count + + signal.recent_edit_count + + signal.recent_read_count + + signal.recent_todowrite_count ) + deep_enough = signal.turn_depth >= _STALL_MIN_TURN_DEPTH + no_production = signal.recent_write_count == 0 and signal.recent_edit_count == 0 + investigating = signal.recent_read_count >= 1 or signal.recent_todowrite_count >= 1 + # spinning vs exploring partition the "not producing" case by investigative + # activity, so at most one fires — no double-counting on the production axis. + spinning = deep_enough and no_production and not investigating + exploring = deep_enough and no_production and investigating return CodingAgentDimensions( severity=float(signal.severity), + spinning=1.0 if spinning else 0.0, + exploring=1.0 if exploring else 0.0, + recent_production_intensity=_ratio( + signal.recent_write_count + signal.recent_edit_count, recent_ops + ), no_error_streak_intensity=_saturating(signal.no_error_streak, scale=3.0), - write_intensity=_ratio(signal.write_count, total_tool_ops), - edit_intensity=_ratio(signal.edit_count, total_tool_ops), - recent_write_intensity=_ratio(signal.recent_write_count, recent_tool_ops), - planning_active=1.0 if signal.recent_todowrite_count > 0 else 0.0, - pure_bash_intensity=_saturating(signal.pure_bash_streak, scale=_PURE_BASH_NORM), - stuck_exploring=1.0 if stuck else 0.0, - no_progress=1.0 if no_progress else 0.0, - # Only treat tests_passed as a signal once the agent has made real changes; - # early test runs against the unmodified codebase are exploratory, not confirmatory. - tests_passed=1.0 if signal.tests_passed and signal.write_count >= 3 else 0.0, ) diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index 362a8f1b..b8435ea3 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -89,28 +89,22 @@ async def _pick( if override is not None: return _record(ctx, decision_log, "override", override) - # Settled run: a recent test-pass backed by a recent code change (write or - # edit) is safe to run on the cheap tier — EFFICIENT for both pickers. - # Windowed, so it lapses once the run moves on. - if signal.tests_passed and (signal.recent_write_count + signal.recent_edit_count) >= 1: + # Settled run: a recent test-pass backed by a recent code change (write or edit), + # with no error in the window — safe on the cheap tier for both pickers. The + # severity gate keeps a windowed HARD error from being swallowed as "settled"; + # such a turn falls through to the scorer, where the error routes it to CAPABLE. + if ( + signal.tests_passed + and (signal.recent_write_count + signal.recent_edit_count) >= 1 + and not signal.severity > 0.0 + ): return _record(ctx, decision_log, "tests_passed", EFFICIENT) dimensions = from_signal(signal) - # efficient_first is weak-by-default, so it escalates to CAPABLE on ANY wrong - # signal (error / stuck / no-progress) before scoring. Without this, a soft - # error — or an error diluted by a co-occurring progress signal (a write-heavy - # turn that also errored nets to ~0) — falls below the confidence bar and drops - # to the EFFICIENT default, leaving a failing turn on the weak tier. capable_first - # already defaults to CAPABLE, so it needs no such bias. - if default_tier == EFFICIENT and ( - dimensions.severity > 0.0 - or dimensions.stuck_exploring > 0.0 - or dimensions.no_progress > 0.0 - ): - return _record(ctx, decision_log, "ef_escalate", CAPABLE) - - # Fixed weights; confidence_threshold is the corroboration dial - # (signals-to-clear = threshold / signal unit). + # Both pickers share the scorer: severity / spinning / exploring corroborate + # toward CAPABLE, production / clean-streak toward EFFICIENT. The + # confidence_threshold is the corroboration dial (see scorer docstring); the two + # pickers differ only in the low-confidence default tier below. result = score(dimensions, weights=weights) if result.confidence >= confidence_threshold: tier = CAPABLE if result.score > 0 else EFFICIENT diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py index 00a470a1..ca5c2690 100644 --- a/switchyard/lib/processors/stage_router/scorer.py +++ b/switchyard/lib/processors/stage_router/scorer.py @@ -1,7 +1,26 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Weighted linear scorer, tanh-squashed to ``(-1, +1)``; confidence = ``abs(score)``.""" +"""Signed linear scorer over two axes (error, production), tanh-squashed to ``(-1, +1)``. + +Each signal contributes a fixed weight; WRONG signals push toward CAPABLE (+), +PROGRESS signals toward EFFICIENT (−). The raw weighted sum is squashed with +``tanh(gain * raw)`` and ``confidence = abs(score)``. + +**On the threshold (read this before tuning):** the raw sum is small — one maxed +signal is ``±0.10``, two corroborating signals ``±0.20``. The gain spreads that +into a usable confidence range; it does **not** make the threshold an integer +"signals-to-clear" count. Empirically the dial reads: + + conf 0.245 exploring alone (half weight) + conf 0.462 one full signal (a HARD error, or spinning) + conf 0.635 severity + exploring + conf 0.762 two full signals (severity + spinning) + +So a threshold near 0.3 escalates on ~one signal, ~0.5 needs ~1.5, ~0.7 needs two +to corroborate. The reachable range is ``(0, ~0.76)`` for these two axes — a +threshold above ~0.76 can't be met, so it always defers to the classifier/default. +""" from __future__ import annotations @@ -11,59 +30,48 @@ from switchyard.lib.processors.stage_router.dimensions import CodingAgentDimensions -#: Gain applied before the tanh squash. The raw weighted sum is small (typical -#: multi-signal turns land near ±0.35), so ``tanh(gain * raw)`` spreads it across -#: most of ``(-1, +1)``. That turns the symmetric ``confidence_threshold`` into a -#: full-range dial: users pick a positive ``t`` in ``(0, 1)`` and the confident -#: fraction sweeps smoothly (e.g. t=0.1 → most turns, t=0.7 → few). Monotonic, so -#: routing sign is unchanged — only the confidence scale is spread out. +#: Gain applied before the tanh squash — spreads the small raw sum across the +#: usable confidence range (see the module docstring for the resulting dial). Load- +#: bearing: without it, confidence would cap near ±0.20 and mid/high thresholds +#: would be unreachable. _SCORE_GAIN: float = 5.0 -#: Max severity that reaches the scorer: `1.0` (critical) is caught by the picker -#: override, so `0.7` (hard) is the strongest error the linear scorer ever sees. +#: Max severity the scorer ever sees: ``1.0`` (critical) is caught by the picker +#: override, so ``0.7`` (hard) is the strongest error that reaches the linear sum. +#: Used to normalise ``severity`` so a HARD error contributes exactly ``_SIGNAL_UNIT``. _HARD_SEVERITY: float = 0.7 -#: Fixed weight one maxed signal contributes. Weights are constant (independent of -#: threshold), so the ``confidence_threshold`` is the sole corroboration dial: -#: ``signals-to-clear = threshold / _SIGNAL_UNIT``. With unit 0.10 the threshold -#: maps to whole signal counts — 0.10 → one maxed signal clears, 0.20 → two must -#: agree, etc. (unit and threshold are redundant for routing; only their ratio -#: matters, so tuning lives entirely in the threshold). Kept well under 1 so the -#: summed score never saturates to ±1 — no single axis can peg the decision. +#: Weight one maxed signal contributes. Small enough that no single axis pegs the +#: decision; corroboration across the two axes is what pushes confidence up. _SIGNAL_UNIT: float = 0.10 -#: "Something is wrong" signals — always push toward CAPABLE (strong), both pickers. -#: severity is per-turn; the other two are windowed gates — all self-clear. -_WRONG_SIGNALS: tuple[str, ...] = ("severity", "stuck_exploring", "no_progress") -#: "Progress / settled" signals — push toward EFFICIENT (weak), both pickers. -#: capable_first / efficient_first differ only in the fall_open default. All windowed. -_PROGRESS_SIGNALS: tuple[str, ...] = ( - "recent_write_intensity", - "planning_active", - "pure_bash_intensity", - "no_error_streak_intensity", -) -#: Max value each signal reaches at the scorer; used to normalise so a maxed -#: signal contributes exactly ``_SIGNAL_UNIT``. Defaults to 1.0 for [0, 1] gates. +#: "Something is wrong" → CAPABLE (strong). ``exploring`` is neutral (half weight): +#: it never escalates alone at a sane threshold, only when corroborated. +_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring") +#: "Making progress" → EFFICIENT (weak). Both are the good poles of the two axes. +_PROGRESS_SIGNALS: tuple[str, ...] = ("recent_production_intensity", "no_error_streak_intensity") +#: Max value each signal reaches, used to normalise so a maxed signal contributes +#: ``_SIGNAL_UNIT``. Defaults to 1.0 for ``[0, 1]`` gates/ratios. _MAX_VALUE: Mapping[str, float] = {"severity": _HARD_SEVERITY} +#: Signals weighted at half unit — deliberately weak, needs corroboration to matter. +_HALF_WEIGHT: frozenset[str] = frozenset({"exploring"}) def _build_weights(unit: float = _SIGNAL_UNIT) -> dict[str, float]: - """Signed, fixed linear weights: wrong → CAPABLE (+), progress → EFFICIENT (-). + """Signed, fixed linear weights: wrong → CAPABLE (+), progress → EFFICIENT (−). - A single maxed signal contributes ``unit`` (severity is normalised by its - 0.7 cap so it too lands at ``unit``). Because every weight is well under 1, - the summed score never saturates; corroboration is set downstream by the - ``confidence_threshold`` (signals-to-clear = threshold / unit). + A maxed signal contributes ``unit`` (``severity`` is normalised by its HARD cap + so it too lands at ``unit``); ``exploring`` is halved to keep it neutral. """ weights: dict[str, float] = {} for name in _WRONG_SIGNALS: - weights[name] = unit / _MAX_VALUE.get(name, 1.0) + w = unit / _MAX_VALUE.get(name, 1.0) + weights[name] = w / 2.0 if name in _HALF_WEIGHT else w for name in _PROGRESS_SIGNALS: weights[name] = -unit / _MAX_VALUE.get(name, 1.0) return weights -#: Fixed scorer weights. Corroboration is dialed by the confidence_threshold. +#: Fixed scorer weights. Corroboration across axes is dialed by confidence_threshold. DEFAULT_WEIGHTS: Mapping[str, float] = _build_weights() @@ -84,9 +92,8 @@ def score( """Score ``dimensions``; raw weighted sum is tanh-squashed into ``(-1, +1)``. ``contributions`` are the raw per-signal weighted values (pre-squash, so they - sum to the raw score); ``score`` is ``tanh(gain * raw)``. The squash spreads - the small raw sum across the range so the ``confidence_threshold`` reads as a - symmetric ``(0, 1)`` dial. + sum to the raw score); ``score`` is ``tanh(gain * raw)``; ``confidence`` its + magnitude. Positive ``score`` → CAPABLE, negative → EFFICIENT. """ contributions: dict[str, float] = {} raw = 0.0 diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py index 863bb09c..74a5139c 100644 --- a/tests/test_stage_router_pickers.py +++ b/tests/test_stage_router_pickers.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Behavioural tests for the stage_router pickers.""" +"""Behavioural tests for the stage_router pickers. + +The two pickers share every decision path (override → tests_passed → scorer → +classifier) and differ only in the low-confidence default tier. +""" from dataclasses import dataclass from typing import Any @@ -34,6 +38,11 @@ def _msg_tool_call(name: str) -> dict: return {"role": "assistant", "tool_calls": [{"function": {"name": name, "arguments": "{}"}}]} +def _msg_bash(cmd: str) -> dict: + args = f'{{"command": "{cmd}"}}' + return {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": args}}]} + + def _msg_tool_result(content: str) -> dict: return {"role": "tool", "tool_call_id": "x", "content": content} @@ -71,11 +80,7 @@ async def acompletion( def _stub_classifier(tier: str | None) -> TierClassifier: - return TierClassifier( - model="stub", - api_key="stub", - client=_StubLLMClient(tier), - ) + return TierClassifier(model="stub", api_key="stub", client=_StubLLMClient(tier)) @pytest.mark.asyncio @@ -85,12 +90,16 @@ async def test_critical_severity_overrides_to_capable(): {"role": "user", "content": "retry"}, ]) assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx([ + _msg_tool_result("Out of memory: cannot allocate memory"), + {"role": "user", "content": "retry"}, + ]) assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE @pytest.mark.asyncio async def test_tests_passed_with_edits_routes_to_efficient(): - """A settled run (recent test-pass + recent edit) is safe on the cheap tier.""" + """A settled run (recent test-pass + recent edit, no error) is safe on cheap tier.""" messages = [_msg_tool_call("Edit")] * 3 + [_msg_tool_result("all tests passed")] * 3 messages += [{"role": "user", "content": "ok continue"}] * 12 ctx = await _ctx(messages) @@ -100,27 +109,55 @@ async def test_tests_passed_with_edits_routes_to_efficient(): @pytest.mark.asyncio -async def test_efficient_first_escalates_on_any_error(): - """efficient_first escalates to CAPABLE on any wrong signal — even a low-confidence - one that would otherwise fall through to the EFFICIENT default — so a failing turn - never stays on the weak tier. capable_first is unaffected (already CAPABLE-default).""" +async def test_tests_passed_blocked_by_windowed_error(): + """A HARD error in the window blocks the settled-run shortcut (finding G) — the + turn falls through to the scorer instead of being swallowed as EFFICIENT.""" log = StageRouterDecisionLog() - ctx = await _ctx([ - _msg_tool_call("Bash"), - _msg_tool_result("bash: foo: command not found"), - {"role": "user", "content": "retry"}, - ]) - # a single soft error is below the confidence bar, yet ef still escalates - assert await pick_efficient_first(ctx, confidence_threshold=0.30, decision_log=log) == CAPABLE - assert ctx.metadata[CONTEXT_KEY] == "ef_escalate" - # the bias is one-sided: capable_first keeps its own path, still CAPABLE - ctx = await _ctx([ - _msg_tool_call("Bash"), - _msg_tool_result("bash: foo: command not found"), - {"role": "user", "content": "retry"}, - ]) - assert await pick_capable_first(ctx, confidence_threshold=0.30) == CAPABLE - assert ctx.metadata[CONTEXT_KEY] != "ef_escalate" + messages = [_msg_tool_call("Edit")] * 3 + [ + _msg_tool_result("all tests passed"), + _msg_tool_result("Traceback (most recent call last):\n ValueError: bad"), + ] + ctx = await _ctx(messages) + await pick_capable_first(ctx, confidence_threshold=0.2, decision_log=log) + assert ctx.metadata[CONTEXT_KEY] != "tests_passed" + + +@pytest.mark.asyncio +async def test_dimensions_routes_capable_on_corroborated_wrong_signals(): + """Deep pure-command cycling (spinning) that ends in an error corroborates + severity + spinning → CAPABLE by sign, for both pickers, no classifier needed.""" + log = StageRouterDecisionLog() + classifier = _stub_classifier(tier="efficient") # would flip if consulted + seq: list[dict] = [] + for i in range(5): + seq.append(_msg_bash("make")) + seq.append(_msg_tool_result( + "Traceback (most recent call last):\n ValueError" if i == 4 else "building..." + )) + seq.append({"role": "user", "content": "next"}) + ctx = await _ctx(seq) + tier = await pick_capable_first( + ctx, confidence_threshold=0.2, classifier=classifier, decision_log=log, + ) + assert ctx.metadata[CONTEXT_KEY] == "dimensions" + assert tier == CAPABLE + assert classifier._client.calls == 0 # type: ignore[attr-defined] + ctx = await _ctx(seq) + assert await pick_efficient_first(ctx, confidence_threshold=0.2) == CAPABLE + + +@pytest.mark.asyncio +async def test_dimensions_routes_efficient_on_progress(): + """A deep, clean, write-heavy run scores negative → EFFICIENT for both pickers.""" + seq: list[dict] = [] + for _ in range(5): + seq.append(_msg_tool_call("Write")) + seq.append(_msg_tool_result("ok")) + seq.append({"role": "user", "content": "next"}) + ctx = await _ctx(seq) + assert await pick_capable_first(ctx, confidence_threshold=0.2) == EFFICIENT + ctx = await _ctx(seq) + assert await pick_efficient_first(ctx, confidence_threshold=0.2) == EFFICIENT @pytest.mark.asyncio @@ -133,8 +170,7 @@ async def test_no_signal_returns_default_tier(): @pytest.mark.asyncio async def test_low_confidence_consults_classifier_when_configured(): """Empty trajectory yields confidence 0.0 — picker must call the classifier.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier="efficient") assert await pick_capable_first( ctx, confidence_threshold=0.7, classifier=classifier, @@ -144,17 +180,16 @@ async def test_low_confidence_consults_classifier_when_configured(): @pytest.mark.asyncio async def test_low_confidence_falls_back_to_default_without_classifier(): - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx([{"role": "user", "content": "hi"}]) assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT @pytest.mark.asyncio async def test_classifier_fall_open_on_unknown_tier(): """A malformed classifier verdict must not override the picker default.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier=None) assert await pick_capable_first( ctx, confidence_threshold=0.7, classifier=classifier, @@ -163,52 +198,17 @@ async def test_classifier_fall_open_on_unknown_tier(): @pytest.mark.asyncio async def test_threshold_zero_skips_classifier_entirely(): - """``confidence_threshold=0`` means accept every scorer verdict, however efficient.""" - messages = [{"role": "user", "content": "hi"}] - ctx = await _ctx(messages) + """``confidence_threshold=0`` accepts every scorer verdict, however weak.""" + ctx = await _ctx([{"role": "user", "content": "hi"}]) classifier = _stub_classifier(tier="capable") - # Empty trajectory → scorer near zero, default sign decides; classifier not consulted. await pick_capable_first(ctx, confidence_threshold=0.0, classifier=classifier) assert classifier._client.calls == 0 # type: ignore[attr-defined] -@pytest.mark.asyncio -async def test_dimensions_branch_routes_by_scorer_sign(): - """A corroborated scorer verdict (≥ threshold) bypasses the classifier. - - No single signal clears threshold (weights are corroborative). A long, - write-free read spree maxes three wrong signals at once — ``stuck_exploring``, - ``no_progress`` (turn_depth > 60), and ``severity`` (the last result errors) — - which together clear the threshold, so ``_pick`` takes the dimensions branch - and routes to CAPABLE by sign without consulting the classifier. - """ - log = StageRouterDecisionLog() - classifier = _stub_classifier(tier="efficient") # would push EFFICIENT if reached - read_spree = [] - for i in range(40): - read_spree.append(_msg_tool_call("Read")) - read_spree.append(_msg_tool_result( - "bash: foo: command not found" if i == 39 else "file contents" - )) - read_spree.append({"role": "user", "content": "next"}) - ctx = await _ctx(read_spree) - tier = await pick_capable_first( - ctx, confidence_threshold=0.20, classifier=classifier, decision_log=log, - ) - assert ctx.metadata[CONTEXT_KEY] == "dimensions" - assert tier == CAPABLE # wrong signals → CAPABLE by sign, regardless of picker - assert classifier._client.calls == 0 # type: ignore[attr-defined] - assert log.snapshot()["dimensions"] == 1 - # efficient_first routes the same corroborated wrong verdict to CAPABLE too. - ctx = await _ctx(read_spree) - assert await pick_efficient_first(ctx, confidence_threshold=0.20) == CAPABLE - - @pytest.mark.asyncio async def test_decision_log_counts_sources(): """Each decision path increments exactly one bucket in the shared log.""" log = StageRouterDecisionLog() - # override path: critical severity → CAPABLE, source=override ctx_critical = await _ctx([ _msg_tool_result("Out of memory: cannot allocate memory"), {"role": "user", "content": "retry"}, @@ -216,12 +216,10 @@ async def test_decision_log_counts_sources(): await pick_capable_first(ctx_critical, confidence_threshold=0.7, decision_log=log) assert ctx_critical.metadata[CONTEXT_KEY] == "override" - # fall_open path: low confidence, no classifier → default tier ctx_neutral = await _ctx([{"role": "user", "content": "hi"}]) await pick_capable_first(ctx_neutral, confidence_threshold=0.99, decision_log=log) assert ctx_neutral.metadata[CONTEXT_KEY] == "fall_open" - # classifier path: low confidence, classifier configured → classifier verdict classifier = _stub_classifier(tier="efficient") ctx_class = await _ctx([{"role": "user", "content": "hi"}]) await pick_capable_first( diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py index 6a49445a..205ff460 100644 --- a/tests/test_stage_router_scorer.py +++ b/tests/test_stage_router_scorer.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for the stage-router dimensions + scorer.""" +"""Unit tests for the stage-router dimensions + two-axis scorer.""" from __future__ import annotations @@ -22,121 +22,93 @@ from switchyard_rust.core import ChatRequest, ProxyContext -def _zero_dimensions() -> CodingAgentDimensions: +def _zero() -> CodingAgentDimensions: return CodingAgentDimensions( severity=0.0, + spinning=0.0, + exploring=0.0, + recent_production_intensity=0.0, no_error_streak_intensity=0.0, - write_intensity=0.0, - edit_intensity=0.0, - recent_write_intensity=0.0, - planning_active=0.0, - pure_bash_intensity=0.0, - stuck_exploring=0.0, - no_progress=0.0, - tests_passed=0.0, ) +def _with(**kw: float) -> CodingAgentDimensions: + return CodingAgentDimensions(**{**_zero().__dict__, **kw}) + + def test_zero_signal_scores_to_zero(): - result = score(_zero_dimensions()) + result = score(_zero()) assert result.score == 0.0 assert result.confidence == 0.0 -def test_critical_severity_pushes_toward_capable(): - dims = _zero_dimensions() - dims = CodingAgentDimensions(**{**dims.__dict__, "severity": 1.0}) - result = score(dims) - assert result.score > 0 - assert result.confidence == abs(result.score) +def test_wrong_signals_point_capable(): + assert score(_with(severity=0.7)).score > 0 + assert score(_with(spinning=1.0)).score > 0 + assert score(_with(exploring=1.0)).score > 0 -def test_tests_passed_is_not_scored(): - """tests_passed is routed to the picker's default tier, not scored here.""" - dims = _zero_dimensions() - dims = CodingAgentDimensions(**{**dims.__dict__, "tests_passed": 1.0}) - result = score(dims) - assert result.score == 0.0 +def test_progress_signals_point_efficient(): + assert score(_with(recent_production_intensity=1.0)).score < 0 + assert score(_with(no_error_streak_intensity=1.0)).score < 0 -def test_progress_signal_points_efficient(): - """A progress signal scores negative (→EFFICIENT), picker-independent.""" - dims = CodingAgentDimensions( - **{**_zero_dimensions().__dict__, "recent_write_intensity": 1.0} - ) - assert score(dims).score < 0 +def test_hard_severity_and_spinning_contribute_one_unit(): + # severity is normalised by its HARD cap (0.7), so a HARD error lands at the + # same unit as a maxed boolean signal like spinning. + hard = score(_with(severity=0.7)) + spin = score(_with(spinning=1.0)) + assert hard.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) + assert spin.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) -def test_wrong_signal_points_capable(): - """A wrong signal scores positive (→CAPABLE), picker-independent.""" - dims = CodingAgentDimensions( - **{**_zero_dimensions().__dict__, "stuck_exploring": 1.0} - ) - assert score(dims).score > 0 +def test_exploring_is_half_weight_and_neutral(): + """exploring contributes half a unit → far lower confidence than a full signal, + so it never clears a default threshold alone (only when corroborated).""" + explore = score(_with(exploring=1.0)) + full = score(_with(spinning=1.0)) + assert explore.score > 0 + assert explore.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.05)) # ~0.245 + assert explore.confidence < full.confidence + assert explore.confidence < 0.30 # below a typical threshold → needs corroboration -def test_tanh_spread_and_monotonic_confidence(): - """Raw sum is tanh-squashed: one signal spreads into the range, more agreeing - signals raise confidence monotonically, and realistic turns never saturate. - """ - one = score( - CodingAgentDimensions(**{**_zero_dimensions().__dict__, "stuck_exploring": 1.0}) - ) - assert one.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) # tanh(gain*unit) - assert one.score > 0 # wrong signal → CAPABLE - - two = score( - CodingAgentDimensions(**{ - **_zero_dimensions().__dict__, - "recent_write_intensity": 1.0, - "pure_bash_intensity": 1.0, - }) - ) - assert two.confidence > one.confidence # more signals → more confident - assert two.score < 0 # progress → EFFICIENT - assert abs(two.score) < 1.0 # realistic turns never saturate +def test_corroboration_raises_confidence(): + """Two agreeing wrong signals corroborate to higher confidence than one.""" + one = score(_with(severity=0.7)) + two = score(_with(severity=0.7, spinning=1.0)) + assert two.confidence > one.confidence + assert two.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.20)) # ~0.762 + + +def test_axes_can_cancel_to_neutral(): + """A turn that both errored and produced nets toward zero confidence.""" + dims = _with(severity=0.7, recent_production_intensity=1.0) + result = score(dims) + assert result.confidence < 0.30 # roughly cancels → defers to classifier/default -def test_extreme_weights_saturate_via_tanh(): - """A large raw sum saturates smoothly to ±1 through tanh (no hard clip).""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - high = score(dims, weights={"severity": 5.0}) # raw 5.0 → tanh(25) ≈ 1.0 +def test_tanh_saturates_smoothly_via_weight_override(): + dims = _with(severity=1.0) + high = score(dims, weights={"severity": 5.0}) # raw 5.0 → tanh(25) ≈ 1.0 assert high.score == pytest.approx(1.0) - assert high.confidence == pytest.approx(1.0) low = score(dims, weights={"severity": -5.0}) assert low.score == pytest.approx(-1.0) - assert low.confidence == pytest.approx(1.0) def test_custom_weights_can_invert_decision(): - """Researchers override weights via the call site, not YAML.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - default = score(dims, weights=DEFAULT_WEIGHTS) - inverted = score(dims, weights={"severity": -0.5}) - assert default.score > 0 - assert inverted.score < 0 + dims = _with(severity=1.0) + assert score(dims, weights=DEFAULT_WEIGHTS).score > 0 + assert score(dims, weights={"severity": -0.5}).score < 0 def test_contributions_are_raw_presquash(): - """``contributions`` are pre-squash: they sum to raw, score = tanh(gain*raw).""" - dims = CodingAgentDimensions( - **{**_zero_dimensions().__dict__, "recent_write_intensity": 0.5} - ) + dims = _with(recent_production_intensity=0.5) result = score(dims) raw = sum(result.contributions.values()) assert result.score == pytest.approx(math.tanh(_SCORE_GAIN * raw)) -def test_contributions_exceed_squashed_score(): - """A large raw sum stays unsquashed in contributions while score saturates.""" - dims = CodingAgentDimensions(**{**_zero_dimensions().__dict__, "severity": 1.0}) - result = score(dims, weights={"severity": 5.0}) - raw = sum(result.contributions.values()) - assert raw == 5.0 - assert result.score == pytest.approx(1.0) - assert raw > result.score - - @pytest.mark.asyncio async def test_from_signal_normalises_real_extracted_signal(): """End-to-end: DimensionCollector → ToolResultSignal → CodingAgentDimensions.""" @@ -156,5 +128,7 @@ async def test_from_signal_normalises_real_extracted_signal(): assert signal is not None dims = from_signal(signal) assert 0.0 <= dims.severity <= 1.0 - assert 0.0 <= dims.write_intensity <= 1.0 - assert dims.write_intensity > 0 # we issued one Write call + assert dims.recent_production_intensity > 0 # we issued one Write call + # too shallow (turn_depth < 8) for a stall signal to fire + assert dims.spinning == 0.0 + assert dims.exploring == 0.0 From 7bddaa841fcc3f2d3eaf1f434d0d4e05c896c42e Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 17 Jul 2026 14:24:12 -0700 Subject: [PATCH 06/21] refactor(stage-router): drop no_error_streak; make exploring a full escalation signal Signed-off-by: Sabhatina Selvam --- .../lib/processors/stage_router/dimensions.py | 21 ++++--------- .../lib/processors/stage_router/scorer.py | 31 ++++++++++--------- tests/test_stage_router_scorer.py | 18 +++++------ 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py index 677a6718..d5fc7115 100644 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ b/switchyard/lib/processors/stage_router/dimensions.py @@ -3,10 +3,11 @@ """Scorer-ready view of :class:`ToolResultSignal` — two axes, five signals. -The stage_router scorer models a coding turn on two independent axes: +The stage_router scorer models a coding turn on two axes: -* **error** — did the recent tool results error? ``severity`` (bad pole) vs - ``no_error_streak_intensity`` (good pole). +* **error** — did the recent tool results error? ``severity`` (escalate-only; a + clean-result streak used to sit on this axis but cancelled the windowed error + signal and released escalations too early, so it was removed). * **production** — is the agent producing code? ``spinning`` / ``exploring`` (bad poles) vs ``recent_production_intensity`` (good pole). @@ -21,7 +22,6 @@ from __future__ import annotations -import math from dataclasses import dataclass from typing import TYPE_CHECKING @@ -33,13 +33,6 @@ _STALL_MIN_TURN_DEPTH: int = 8 -def _saturating(x: float, scale: float) -> float: - """Map non-negative counts to ``[0, 1)``; ``scale`` is the half-saturation point.""" - if x <= 0: - return 0.0 - return 1.0 - math.exp(-x / scale) - - def _ratio(numerator: int, denominator: int) -> float: return numerator / denominator if denominator > 0 else 0.0 @@ -55,12 +48,11 @@ class CodingAgentDimensions: #: is only cycling non-inspecting commands (a struggle signal → strong). spinning: float #: 1.0 when the recent window has reads/plans but no writes or edits — the agent - #: is investigating without converging (a neutral-ish signal → strong, half weight). + #: is investigating without converging. Full escalation signal → strong; because it + #: persists while the agent isn't producing, it latches sustained escalation on hard tasks. exploring: float #: Fraction of recent tool ops that produced code (writes + edits) → weak. recent_production_intensity: float - #: Saturating count of consecutive clean recent results → weak. - no_error_streak_intensity: float def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: @@ -95,7 +87,6 @@ def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: recent_production_intensity=_ratio( signal.recent_write_count + signal.recent_edit_count, recent_ops ), - no_error_streak_intensity=_saturating(signal.no_error_streak, scale=3.0), ) diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py index ca5c2690..1e7762c1 100644 --- a/switchyard/lib/processors/stage_router/scorer.py +++ b/switchyard/lib/processors/stage_router/scorer.py @@ -12,10 +12,8 @@ into a usable confidence range; it does **not** make the threshold an integer "signals-to-clear" count. Empirically the dial reads: - conf 0.245 exploring alone (half weight) - conf 0.462 one full signal (a HARD error, or spinning) - conf 0.635 severity + exploring - conf 0.762 two full signals (severity + spinning) + conf 0.462 one full signal (a HARD error, spinning, or exploring) + conf 0.762 two full signals (e.g. severity + exploring, severity + spinning) So a threshold near 0.3 escalates on ~one signal, ~0.5 needs ~1.5, ~0.7 needs two to corroborate. The reachable range is ``(0, ~0.76)`` for these two axes — a @@ -44,28 +42,33 @@ #: decision; corroboration across the two axes is what pushes confidence up. _SIGNAL_UNIT: float = 0.10 -#: "Something is wrong" → CAPABLE (strong). ``exploring`` is neutral (half weight): -#: it never escalates alone at a sane threshold, only when corroborated. +#: "Something is wrong" → CAPABLE (strong). ``exploring`` (reading/planning without +#: producing) is a FULL escalation signal: it clears the threshold on its own, and +#: because the condition persists while the agent isn't producing, it holds strong +#: across the whole stretch — the "latch" that sustains escalation on hard debugging +#: tasks (verified against runs where a struggling agent reads-without-writing for +#: dozens of turns; treating it as neutral routed those to the weak tier and lost them). _WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring") -#: "Making progress" → EFFICIENT (weak). Both are the good poles of the two axes. -_PROGRESS_SIGNALS: tuple[str, ...] = ("recent_production_intensity", "no_error_streak_intensity") +#: "Making progress" → EFFICIENT (weak). De-escalation is driven by real production +#: (writes/edits), not by a clean-result streak: a streak of clean results actively +#: cancelled the (windowed) error signal on the same axis and released escalations too +#: early — trace-verified to suppress escalation entirely on some tasks — so the error +#: axis is now escalate-only (severity) and progress is production. +_PROGRESS_SIGNALS: tuple[str, ...] = ("recent_production_intensity",) #: Max value each signal reaches, used to normalise so a maxed signal contributes #: ``_SIGNAL_UNIT``. Defaults to 1.0 for ``[0, 1]`` gates/ratios. _MAX_VALUE: Mapping[str, float] = {"severity": _HARD_SEVERITY} -#: Signals weighted at half unit — deliberately weak, needs corroboration to matter. -_HALF_WEIGHT: frozenset[str] = frozenset({"exploring"}) def _build_weights(unit: float = _SIGNAL_UNIT) -> dict[str, float]: """Signed, fixed linear weights: wrong → CAPABLE (+), progress → EFFICIENT (−). - A maxed signal contributes ``unit`` (``severity`` is normalised by its HARD cap - so it too lands at ``unit``); ``exploring`` is halved to keep it neutral. + Every maxed signal contributes ``unit`` (``severity`` is normalised by its HARD + cap so it too lands at ``unit``). """ weights: dict[str, float] = {} for name in _WRONG_SIGNALS: - w = unit / _MAX_VALUE.get(name, 1.0) - weights[name] = w / 2.0 if name in _HALF_WEIGHT else w + weights[name] = unit / _MAX_VALUE.get(name, 1.0) for name in _PROGRESS_SIGNALS: weights[name] = -unit / _MAX_VALUE.get(name, 1.0) return weights diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py index 205ff460..df84ee0c 100644 --- a/tests/test_stage_router_scorer.py +++ b/tests/test_stage_router_scorer.py @@ -28,7 +28,6 @@ def _zero() -> CodingAgentDimensions: spinning=0.0, exploring=0.0, recent_production_intensity=0.0, - no_error_streak_intensity=0.0, ) @@ -48,9 +47,8 @@ def test_wrong_signals_point_capable(): assert score(_with(exploring=1.0)).score > 0 -def test_progress_signals_point_efficient(): +def test_progress_signal_points_efficient(): assert score(_with(recent_production_intensity=1.0)).score < 0 - assert score(_with(no_error_streak_intensity=1.0)).score < 0 def test_hard_severity_and_spinning_contribute_one_unit(): @@ -62,15 +60,15 @@ def test_hard_severity_and_spinning_contribute_one_unit(): assert spin.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) -def test_exploring_is_half_weight_and_neutral(): - """exploring contributes half a unit → far lower confidence than a full signal, - so it never clears a default threshold alone (only when corroborated).""" +def test_exploring_is_a_full_escalation_signal(): + """exploring is a full-weight WRONG signal → clears a default threshold on its own + (it's the persistent 'reading-without-producing' latch), same weight as spinning.""" explore = score(_with(exploring=1.0)) - full = score(_with(spinning=1.0)) + spin = score(_with(spinning=1.0)) assert explore.score > 0 - assert explore.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.05)) # ~0.245 - assert explore.confidence < full.confidence - assert explore.confidence < 0.30 # below a typical threshold → needs corroboration + assert explore.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) # ~0.462 + assert explore.confidence == pytest.approx(spin.confidence) + assert explore.confidence > 0.30 # escalates alone at a typical threshold def test_corroboration_raises_confidence(): From 060a8a64c2c088a66e85c4b4d27c3e4f8cd9e524 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Mon, 20 Jul 2026 09:29:37 -0700 Subject: [PATCH 07/21] feat(stage_router): escalate on context compaction + repeated-command churn Signed-off-by: Sabhatina Selvam --- .../src/dimension_collector/tool_signals.rs | 125 +++++++++++++++++- .../component_bindings/dimension_collector.rs | 14 ++ .../lib/processors/stage_router/dimensions.py | 5 + .../lib/processors/stage_router/picker.py | 11 +- .../lib/processors/stage_router/scorer.py | 2 +- switchyard_rust/components.pyi | 2 + tests/test_stage_router_pickers.py | 15 +++ tests/test_stage_router_scorer.py | 9 ++ 8 files changed, 175 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index 9acf8393..c3933514 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -8,6 +8,9 @@ //! [`switchyard_protocol::Request`] (raw body + wire-format metadata) so the two //! request models share one implementation. +use std::collections::HashMap; + +use serde_json::Value; use switchyard_core::{ChatRequest, ChatRequestType}; use switchyard_protocol::{Metadata, Request, WireFormat}; @@ -222,6 +225,17 @@ pub struct ToolResultSignal { pub turn_depth: u32, /// Char count of the last user message. pub prompt_char_count: u32, + /// Over the recent window, the largest share taken by a single identical + /// command-bearing tool call (Bash name + command), in `[0, 1]`. A weak model + /// looping the same command (e.g. `cd /tmp` many times) spikes this even when it + /// errors little and keeps "producing" — churn the error/stall signals miss. + pub repeated_cmd_ratio: f32, + /// The request carries a context-compaction summary (the agent's context was + /// summarised after overflowing). Compaction resets the router's accumulated + /// signals, so a task that was on the strong tier de-escalates back to weak — the + /// picker uses this to force + hold the strong tier. Self-latching: the summary + /// stays in the context prefix on every subsequent turn. + pub compacted: bool, } /// Extract tool-execution signals using the default `recent_*` window. @@ -239,14 +253,14 @@ pub fn extract_tool_signals_with_window( return ToolResultSignal::default(); }; - match request.request_type() { + let (mut signal, entries) = match request.request_type() { ChatRequestType::Anthropic => { let messages = obj .get("messages") .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_anthropic(messages, recent_window) + (extract_from_messages_anthropic(messages, recent_window), messages) } ChatRequestType::OpenAiResponses => { let items = obj @@ -254,7 +268,7 @@ pub fn extract_tool_signals_with_window( .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_input_responses(items, recent_window) + (extract_from_input_responses(items, recent_window), items) } ChatRequestType::OpenAiChat => { let messages = obj @@ -262,13 +276,40 @@ pub fn extract_tool_signals_with_window( .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_openai_chat(messages, recent_window) + (extract_from_messages_openai_chat(messages, recent_window), messages) } - } + }; + + // Compaction is detected anywhere in the message/item contents (the summary stays + // in the prefix on every subsequent turn, so this self-latches once it fires). + signal.compacted = entries.iter().any(|m| { + m.as_object() + .is_some_and(|o| content_has_compaction_marker(o.get("content"))) + }); + signal } // ─── format-specific extractors ────────────────────────────────────────────── +/// Distinctive preamble Claude Code injects as a user message when it compacts an +/// overflowed context. Matched case-insensitively; normal task text never contains it. +const COMPACTION_MARKER: &str = "session is being continued"; + +/// True when a user-message content (string or text blocks) carries the compaction +/// summary preamble. +fn content_has_compaction_marker(content: Option<&Value>) -> bool { + match content { + Some(Value::String(s)) => s.to_lowercase().contains(COMPACTION_MARKER), + Some(Value::Array(blocks)) => blocks.iter().any(|b| { + b.as_object() + .and_then(|o| o.get("text")) + .and_then(Value::as_str) + .is_some_and(|t| t.to_lowercase().contains(COMPACTION_MARKER)) + }), + _ => false, + } +} + fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolResultSignal { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); @@ -540,6 +581,23 @@ fn build_signal( let tests_passed = detect_tests_passed(&tool_texts, recent_window); + // repeated_cmd_ratio: over the recent window, the largest count of an identical + // command-bearing call (Bash name + command) / window. Catches a weak model + // looping the same command with no progress — the churn that severity/spinning + // miss because it errors little and still "produces". Only command-bearing calls + // count, so distinct reads/writes don't false-positive. + let mut cmd_counts: HashMap<(&str, &str), u32> = HashMap::new(); + for tc in tool_calls.iter().skip(recent_start) { + if let Some(cmd) = tc.command.as_deref() { + *cmd_counts.entry((tc.name.as_str(), cmd)).or_insert(0) += 1; + } + } + let repeated_cmd_ratio = if recent_window > 0 { + cmd_counts.values().copied().max().unwrap_or(0) as f32 / recent_window as f32 + } else { + 0.0 + }; + ToolResultSignal { severity, patterns, @@ -556,6 +614,10 @@ fn build_signal( tests_passed, turn_depth, prompt_char_count, + repeated_cmd_ratio, + // Set by extract_tool_signals_with_window after the format-specific extract, + // which scans all message contents for the compaction marker. + compacted: false, } } @@ -875,6 +937,59 @@ mod tests { assert_eq!(wide.recent_edit_count, 1); } + #[test] + fn repeated_cmd_ratio_spikes_on_a_looping_command() { + // Five identical `cd /tmp` calls fill a window of 5 → ratio 1.0 (churn). + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, + ] + })); + assert_eq!(extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, 1.0); + } + + #[test] + fn compaction_marker_sets_compacted() { + // The compaction summary is a user message carrying Claude Code's preamble. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "user", "content": "This session is being continued from a previous conversation that ran out of context."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(extract_tool_signals(&request).compacted); + } + + #[test] + fn no_compaction_marker_stays_uncompacted() { + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "user", "content": "Write a script that parses the log file."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(!extract_tool_signals(&request).compacted); + } + + #[test] + fn repeated_cmd_ratio_stays_low_for_distinct_commands() { + // Five distinct commands → max repeat 1 / window 5 = 0.2, no false churn. + let request = ChatRequest::openai_chat(json!({ + "messages": [ + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls a\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls b\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls c\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls d\"}"}}]}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls e\"}"}}]}, + ] + })); + assert_eq!(extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, 0.2); + } + #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. diff --git a/crates/switchyard-py/src/component_bindings/dimension_collector.rs b/crates/switchyard-py/src/component_bindings/dimension_collector.rs index 026d193c..5fd8539f 100644 --- a/crates/switchyard-py/src/component_bindings/dimension_collector.rs +++ b/crates/switchyard-py/src/component_bindings/dimension_collector.rs @@ -541,6 +541,20 @@ impl PyToolResultSignal { self.inner.prompt_char_count } + /// Largest share taken by a single identical command-bearing tool call over the + /// recent window, in `[0, 1]` — a weak model looping the same command (churn). + #[getter] + fn repeated_cmd_ratio(&self) -> f32 { + self.inner.repeated_cmd_ratio + } + + /// The request carries a context-compaction summary — the picker forces + holds + /// the strong tier, since compaction otherwise de-escalates the router to weak. + #[getter] + fn compacted(&self) -> bool { + self.inner.compacted + } + fn __repr__(&self) -> String { format!( "ToolResultSignal(severity={:.2}, streak={}, edit={}, write={}, tests_passed={})", diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py index d5fc7115..6c1db768 100644 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ b/switchyard/lib/processors/stage_router/dimensions.py @@ -53,6 +53,10 @@ class CodingAgentDimensions: exploring: float #: Fraction of recent tool ops that produced code (writes + edits) → weak. recent_production_intensity: float + #: Largest share of the recent window taken by one identical command-bearing call + #: in ``[0, 1]`` — a weak model looping the same command (churn) → strong. Catches + #: the "busy but not converging" case that severity/spinning miss. + repeated_cmd_ratio: float def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: @@ -87,6 +91,7 @@ def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: recent_production_intensity=_ratio( signal.recent_write_count + signal.recent_edit_count, recent_ops ), + repeated_cmd_ratio=float(signal.repeated_cmd_ratio), ) diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index b8435ea3..580377bf 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -140,9 +140,16 @@ def _record( def _apply_overrides(signal: "ToolResultSignal") -> int | None: """Non-negotiable, signal-derived shortcuts that bypass the scorer. - A CRITICAL severity always forces CAPABLE; it outranks the settled-run - (`tests_passed`) shortcut handled in :func:`_pick`. + A CRITICAL severity or a context compaction always forces CAPABLE; both outrank + the settled-run (`tests_passed`) shortcut handled in :func:`_pick`. + + Compaction resets the router's accumulated signals, so a task that had escalated + de-escalates straight back to weak. Once the context has been compacted we force — + and, because the summary stays in the prefix on every subsequent turn, hold — the + strong tier, which is where a task hard enough to overflow its context belongs. """ + if signal.compacted: + return CAPABLE if signal.severity >= SEVERITY_CRITICAL: return CAPABLE return None diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py index 1e7762c1..35dbfd21 100644 --- a/switchyard/lib/processors/stage_router/scorer.py +++ b/switchyard/lib/processors/stage_router/scorer.py @@ -48,7 +48,7 @@ #: across the whole stretch — the "latch" that sustains escalation on hard debugging #: tasks (verified against runs where a struggling agent reads-without-writing for #: dozens of turns; treating it as neutral routed those to the weak tier and lost them). -_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring") +_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring", "repeated_cmd_ratio") #: "Making progress" → EFFICIENT (weak). De-escalation is driven by real production #: (writes/edits), not by a clean-result streak: a streak of clean results actively #: cancelled the (windowed) error signal on the same axis and released escalations too diff --git a/switchyard_rust/components.pyi b/switchyard_rust/components.pyi index 01621be5..7facbbf7 100644 --- a/switchyard_rust/components.pyi +++ b/switchyard_rust/components.pyi @@ -347,6 +347,8 @@ class ToolResultSignal: no_error_streak: int tests_passed: bool prompt_char_count: int + repeated_cmd_ratio: float + compacted: bool def get_tool_result_signal(ctx: ProxyContext) -> ToolResultSignal | None: ... diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py index 74a5139c..dc8a49f8 100644 --- a/tests/test_stage_router_pickers.py +++ b/tests/test_stage_router_pickers.py @@ -97,6 +97,21 @@ async def test_critical_severity_overrides_to_capable(): assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE +@pytest.mark.asyncio +async def test_compaction_overrides_to_capable(): + """A compacted context forces CAPABLE for both pickers, even with a production + signal that would otherwise route EFFICIENT — a task hard enough to overflow its + context belongs on strong, and the override holds it there.""" + msgs = [ + {"role": "user", "content": + "This session is being continued from a previous conversation that ran out of context."}, + ] + [_msg_tool_call("Write"), _msg_tool_result("ok")] * 3 + ctx = await _ctx(msgs) + assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE + ctx = await _ctx(msgs) + assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE + + @pytest.mark.asyncio async def test_tests_passed_with_edits_routes_to_efficient(): """A settled run (recent test-pass + recent edit, no error) is safe on cheap tier.""" diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py index df84ee0c..46ed8107 100644 --- a/tests/test_stage_router_scorer.py +++ b/tests/test_stage_router_scorer.py @@ -28,6 +28,7 @@ def _zero() -> CodingAgentDimensions: spinning=0.0, exploring=0.0, recent_production_intensity=0.0, + repeated_cmd_ratio=0.0, ) @@ -71,6 +72,14 @@ def test_exploring_is_a_full_escalation_signal(): assert explore.confidence > 0.30 # escalates alone at a typical threshold +def test_repeated_cmd_ratio_is_a_wrong_signal(): + """A weak model looping one command (repeated_cmd_ratio→1) is a full WRONG signal + → CAPABLE, same weight as spinning. Catches churn that severity/spinning miss.""" + churn = score(_with(repeated_cmd_ratio=1.0)) + assert churn.score > 0 + assert churn.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) + + def test_corroboration_raises_confidence(): """Two agreeing wrong signals corroborate to higher confidence than one.""" one = score(_with(severity=0.7)) From 7d68caa1fb60d273c2c2eaff7fabaf872db9d598 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Mon, 20 Jul 2026 13:18:53 -0700 Subject: [PATCH 08/21] feat(stage_router): optional tier-transition handoff notes (off by default, configurable) Signed-off-by: Sabhatina Selvam --- ...outer-evalsnem-opus-ef-t30-w5-handoff.yaml | 37 +++ switchyard/cli/route_bundle.py | 1 + .../lib/processors/stage_router/__init__.py | 6 + .../processors/stage_router/handoff_notes.py | 218 ++++++++++++++++++ .../stage_router_request_processor.py | 10 + switchyard/lib/profiles/stage_router.py | 14 ++ .../lib/profiles/stage_router_config.py | 31 +++ tests/test_stage_router_handoff_notes.py | 139 +++++++++++ 8 files changed, 456 insertions(+) create mode 100644 benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml create mode 100644 switchyard/lib/processors/stage_router/handoff_notes.py create mode 100644 tests/test_stage_router_handoff_notes.py diff --git a/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml b/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml new file mode 100644 index 00000000..c7f8b06f --- /dev/null +++ b/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml @@ -0,0 +1,37 @@ +# A/B variant of tb2-1-stage-router-evalsnem-opus-ef-t30-w5 with tier-transition +# handoff notes ENABLED (escalation direction only; de-escalation left off). +# Everything else — ef · evals-nemotron weak · Opus 4.8 @ high · window 5 · t0.30 — +# is identical, so a diff vs the base run isolates the note's effect. +routes: + switchyard: + type: stage_router + picker: efficient_first + confidence_threshold: 0.30 + signal_recent_window: 5 + fallback_target_on_evict: strong + enable_stats: true + handoff_notes: + enabled: true + only_on_wrong_signal_escalation: true + # escalation_note uses the built-in default; override here to customise. + # deescalation_note: left unset (strong→weak hand-backs inject nothing). + strong: + model: azure/anthropic/claude-opus-4-8 + api_key: ${OPENAI_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: anthropic + timeout_secs: 600 + extra_body: + output_config: + effort: high + weak: + model: nvidia/nvidia/evals-nemotron-ultra-3 + api_key: ${OPENAI_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + extra_headers: + X-Inference-Priority: batch + extra_body: + chat_template_kwargs: + enable_thinking: true diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 82e14254..21d6a7eb 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -240,6 +240,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "confidence_threshold", "signal_recent_window", "classifier", + "handoff_notes", "enable_stats", "fallback_target_on_evict", }) diff --git a/switchyard/lib/processors/stage_router/__init__.py b/switchyard/lib/processors/stage_router/__init__.py index 217acc7d..358633ba 100644 --- a/switchyard/lib/processors/stage_router/__init__.py +++ b/switchyard/lib/processors/stage_router/__init__.py @@ -17,6 +17,10 @@ CodingAgentDimensions, from_signal, ) +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_ESCALATION_NOTE, + HandoffNoteInjector, +) from switchyard.lib.processors.stage_router.picker import ( CAPABLE, EFFICIENT, @@ -31,9 +35,11 @@ __all__ = [ "CONTEXT_KEY", + "DEFAULT_ESCALATION_NOTE", "DEFAULT_WEIGHTS", "CAPABLE", "CAPABLE_TIER", + "HandoffNoteInjector", "StageRouterDecisionLog", "CodingAgentDimensions", "DecisionSource", diff --git a/switchyard/lib/processors/stage_router/handoff_notes.py b/switchyard/lib/processors/stage_router/handoff_notes.py new file mode 100644 index 00000000..7bb8ccdf --- /dev/null +++ b/switchyard/lib/processors/stage_router/handoff_notes.py @@ -0,0 +1,218 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Optional tier-transition handoff notes. + +When the stage_router flips a session between the efficient (weak) and capable +(strong) tiers, this injector appends a short, ephemeral guidance note to the +outgoing request so the model taking over knows *why* it was handed the task. + +**Ephemeral by construction.** The note is appended to the request the proxy +forwards for this turn only; it is never written back into the agent's +conversation store. On the next turn the agent re-sends its own history (which +never contained the note) and the injector decides afresh. So notes never +accumulate across turns — a task that escalates, de-escalates, then escalates +again gets at most one note per *transition*, each living in a single request. + +**Transition-gated.** A note is injected only when the picked tier *differs* +from the tier this session used on its previous turn (tracked per session in +:attr:`_last_tier`). Staying on a tier — however many turns — injects nothing, +which keeps the note out of the prompt-cache prefix on the steady-state turns. + +**Truthful escalation gate.** With ``only_on_wrong_signal_escalation`` (default), +the escalation note fires only when the escalation was driven by a real signal +(``override`` — critical severity or compaction — or ``dimensions`` — the scorer +crossing the threshold on WRONG signals), not by an ambiguous default +(``fall_open`` / ``llm-classifier``). This keeps "the previous model was +stalling" from being asserted when no such signal was seen. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from switchyard.lib.processors.stage_router.picker import CAPABLE, EFFICIENT + +if TYPE_CHECKING: + from switchyard_rust.core import ChatRequest + +log = logging.getLogger(__name__) + +#: Default weak→strong note. Kept to two sentences: state the reason, then the +#: corrective (don't blindly repeat the weak model's last approach). +DEFAULT_ESCALATION_NOTE: str = ( + "[router-guidance] A weaker model was handling this task and showed signs of " + "stalling, looping, or repeated errors on the preceding steps, so control was " + "escalated to you, a stronger model. Re-examine the current state directly and " + "do not simply repeat the previous approach." +) + +#: Default strong→weak note (only used when a deescalation note is configured; +#: the direction is off unless the operator sets one). +DEFAULT_DEESCALATION_NOTE: str = ( + "[router-guidance] A stronger model just completed the hard reasoning for this " + "task; the remaining work is expected to be mechanical. Execute the established " + "plan and avoid re-architecting what is already in place." +) + +#: Decision sources that represent a *real* signal-driven escalation (vs. an +#: ambiguous default). Used by the escalation gate. +_WRONG_ESCALATION_SOURCES: frozenset[str] = frozenset({"override", "dimensions"}) + +#: Cap on the per-session last-tier map so a long-lived proxy process doesn't +#: grow it without bound. When exceeded the whole map is cleared (cheap, and the +#: only cost is a possible missed note on the first turn of in-flight sessions). +_MAX_SESSIONS: int = 8192 + + +class HandoffNoteInjector: + """Appends a tier-transition guidance note to the outgoing request. + + One instance per stage_router processor; safe under the single-threaded + asyncio loop (the per-session map is only touched between awaits). + """ + + def __init__( + self, + *, + escalation_note: str = DEFAULT_ESCALATION_NOTE, + deescalation_note: str | None = None, + only_on_wrong_signal_escalation: bool = True, + max_sessions: int = _MAX_SESSIONS, + ) -> None: + self._escalation_note = escalation_note + self._deescalation_note = deescalation_note + self._only_on_wrong_signal = only_on_wrong_signal_escalation + self._max_sessions = max_sessions + self._last_tier: dict[str, int] = {} + + def maybe_inject( + self, + request: ChatRequest, + *, + tier: int, + source: str | None, + ) -> bool: + """Inject a note if this turn crosses a tier boundary for its session. + + Returns ``True`` iff a note was appended to ``request``. Never raises — + any failure degrades to "no note" so it can't block routing. + """ + try: + key = _session_key(request) + if key is None: + return False + prev = self._last_tier.get(key) + self._remember(key, tier) + if prev is None or prev == tier: + return False # first turn of the session, or no transition + + note = self._note_for_transition(prev, tier, source) + if not note: + return False + return _append_note(request, note) + except Exception: + log.debug("handoff-note injection failed; continuing without note", exc_info=True) + return False + + def _note_for_transition(self, prev: int, tier: int, source: str | None) -> str | None: + if prev == EFFICIENT and tier == CAPABLE: + if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: + return None + return self._escalation_note + if prev == CAPABLE and tier == EFFICIENT: + return self._deescalation_note + return None + + def _remember(self, key: str, tier: int) -> None: + if len(self._last_tier) >= self._max_sessions and key not in self._last_tier: + self._last_tier.clear() + self._last_tier[key] = tier + + +def _session_key(request: ChatRequest) -> str | None: + """Stable per-session fingerprint from the conversation's fixed prefix. + + Claude Code (and agentic callers generally) keep the system prompt and the + first user turn — the task instruction — constant for the life of a session + while appending turns, so a hash of that prefix identifies the session + without relying on an optional session-id header. + """ + body = getattr(request, "body", None) + if not isinstance(body, dict): + return None + parts: list[str] = [] + system = body.get("system") + if isinstance(system, str): + parts.append(system) + elif isinstance(system, list): + parts.append(_text_of(system)) + messages = body.get("messages") + if isinstance(messages, list) and messages: + first = messages[0] + if isinstance(first, dict): + parts.append(_text_of(first.get("content"))) + if not any(parts): + return None + return str(hash("\x1f".join(parts))) + + +def _text_of(content: object) -> str: + """Flatten message/system content (str or list of blocks) to plain text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + out: list[str] = [] + for block in content: + if isinstance(block, dict): + text = block.get("text") + if isinstance(text, str): + out.append(text) + return "".join(out) + return "" + + +def _append_note(request: ChatRequest, note: str) -> bool: + """Append ``note`` to the request's final user turn. + + Anthropic: add a ``text`` block after the trailing user turn's content + (``tool_result`` blocks must stay first, so the note goes last). OpenAI Chat: + append a fresh trailing ``user`` message (the trailing turn is often a + ``tool`` result, and OpenAI permits a following user message). Both preserve + the cached prefix — only the suffix of the request changes. + """ + body = getattr(request, "body", None) + if not isinstance(body, dict): + return False + messages = body.get("messages") + if not isinstance(messages, list) or not messages: + return False + + fmt = getattr(getattr(request, "request_type", None), "value", None) + block = {"type": "text", "text": note} + + if fmt == "anthropic": + last = messages[-1] + if not isinstance(last, dict) or last.get("role") != "user": + return False + content = last.get("content") + if isinstance(content, list): + content.append(block) + elif isinstance(content, str): + last["content"] = [{"type": "text", "text": content}, block] + else: + return False + else: + # OpenAI Chat / Responses: a trailing user message is the portable append. + messages.append({"role": "user", "content": note}) + + request.replace_body(body) + return True + + +__all__ = [ + "DEFAULT_DEESCALATION_NOTE", + "DEFAULT_ESCALATION_NOTE", + "HandoffNoteInjector", +] diff --git a/switchyard/lib/processors/stage_router_request_processor.py b/switchyard/lib/processors/stage_router_request_processor.py index b91c02d4..ba2b396d 100644 --- a/switchyard/lib/processors/stage_router_request_processor.py +++ b/switchyard/lib/processors/stage_router_request_processor.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from switchyard.lib.backends.llm_target import LlmTarget + from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest @@ -48,6 +49,7 @@ def __init__( picker: TierPicker, classifier: TierClassifier | None = None, decision_log: StageRouterDecisionLog | None = None, + handoff_injector: HandoffNoteInjector | None = None, ) -> None: if len(targets) != 2: raise ValueError(f"stage_router requires exactly 2 targets, got {len(targets)}") @@ -57,6 +59,7 @@ def __init__( self._classifier = classifier self._max_index = len(targets) - 1 self._decision_log = decision_log if decision_log is not None else StageRouterDecisionLog() + self._handoff_injector = handoff_injector self._stats_accumulator: StatsAccumulator | None = None def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None: @@ -83,6 +86,13 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: ctx.selected_target = self._target_ids[idx] ctx.selected_model = self._target_models[idx] await self._record_decision_source(ctx) + if self._handoff_injector is not None: + source = ctx.metadata.get(CONTEXT_KEY) + self._handoff_injector.maybe_inject( + request, + tier=idx, + source=source if isinstance(source, str) else None, + ) log.debug( "stage_router pick: idx=%d target=%s model=%s", idx, ctx.selected_target, ctx.selected_model, diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py index 8429832f..35a7fa43 100644 --- a/switchyard/lib/profiles/stage_router.py +++ b/switchyard/lib/profiles/stage_router.py @@ -10,6 +10,7 @@ from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint from switchyard.lib.processors.stage_router import StageRouterDecisionLog, TierClassifier +from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector from switchyard.lib.processors.stage_router_request_processor import ( BUILTIN_PICKERS, StageRouterRequestProcessor, @@ -18,6 +19,7 @@ from switchyard.lib.profiles.chain import ComponentChainProfile from switchyard.lib.profiles.stage_router_config import ( ClassifierConfig, + HandoffNoteConfig, StageRouterConfig, ) from switchyard.lib.profiles.table import profile_config @@ -53,6 +55,7 @@ def build(self) -> ComponentChainProfile: picker=_build_tier_picker(config, decision_log, classifier), classifier=classifier, decision_log=decision_log, + handoff_injector=_build_handoff_injector(config.handoff_notes), ) ) @@ -83,6 +86,17 @@ def _build_tier_picker( ) +def _build_handoff_injector(config: HandoffNoteConfig | None) -> HandoffNoteInjector | None: + """Build the optional tier-transition note injector; ``None`` when disabled.""" + if config is None or not config.enabled: + return None + return HandoffNoteInjector( + escalation_note=config.escalation_note, + deescalation_note=config.deescalation_note, + only_on_wrong_signal_escalation=config.only_on_wrong_signal_escalation, + ) + + def _build_classifier(config: ClassifierConfig | None) -> TierClassifier | None: """Build the optional LLM fallback classifier for stage_router routing.""" if config is None: diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index ed90e321..7a92a5c8 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -10,6 +10,9 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_ESCALATION_NOTE, +) #: Picker mode accepted in YAML. The profile resolves it to a :class:`TierPicker`. #: The name describes the *default tier* — what the picker returns when the @@ -34,6 +37,32 @@ class ClassifierConfig(BaseModel): recent_turn_window: int = Field(default=3, ge=0) +class HandoffNoteConfig(BaseModel): + """Optional tier-transition guidance notes injected into the request. + + Off by default. When ``enabled``, a short ephemeral note is appended to the + outgoing request on a *tier transition* only (weak↔strong), telling the model + taking over why it was handed the task. Notes are never persisted into the + agent's history, so they do not accumulate across turns. See + ``switchyard.lib.processors.stage_router.handoff_notes``. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + #: Master switch. When ``False`` (default) no note is ever injected, even if + #: the note texts below are customised. + enabled: bool = False + #: Weak→strong note. Injected when a session escalates to the capable tier. + escalation_note: str = DEFAULT_ESCALATION_NOTE + #: Strong→weak note. ``None`` (default) disables the de-escalation direction; + #: set a string to also annotate hand-backs to the efficient tier. + deescalation_note: str | None = None + #: When ``True`` (default) only inject the escalation note when the escalation + #: was driven by a real signal (critical severity / compaction override, or the + #: scorer crossing the threshold), not by an ambiguous low-confidence default. + only_on_wrong_signal_escalation: bool = True + + class StageRouterConfig(BaseModel): """Configuration for the stage-router-routing profile.""" @@ -58,6 +87,8 @@ class StageRouterConfig(BaseModel): #: ``DEFAULT_RECENT_WINDOW`` in ``tool_signals.rs``. signal_recent_window: int = Field(default=3, ge=1) classifier: ClassifierConfig | None = None + #: Optional tier-transition handoff notes (off unless ``handoff_notes.enabled``). + handoff_notes: HandoffNoteConfig | None = None enable_stats: bool = True @field_validator("capable", "efficient", mode="before") diff --git a/tests/test_stage_router_handoff_notes.py b/tests/test_stage_router_handoff_notes.py new file mode 100644 index 00000000..8f84d8ad --- /dev/null +++ b/tests/test_stage_router_handoff_notes.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the optional tier-transition handoff-note injector.""" + +from __future__ import annotations + +from switchyard.lib.processors.stage_router import CAPABLE, EFFICIENT +from switchyard.lib.processors.stage_router.handoff_notes import ( + DEFAULT_DEESCALATION_NOTE, + DEFAULT_ESCALATION_NOTE, + HandoffNoteInjector, +) +from switchyard_rust.core import ChatRequest + + +def _anthropic(task: str = "solve the task", turn: str = "next") -> ChatRequest: + """Anthropic request whose trailing user turn carries a tool_result block.""" + return ChatRequest.anthropic({ + "system": "you are a coding agent", + "messages": [ + {"role": "user", "content": task}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": turn}]}, + ], + }) + + +def _last_user_text(request: ChatRequest) -> str: + content = request.body["messages"][-1]["content"] + if isinstance(content, str): + return content + return "".join(b.get("text", "") for b in content if isinstance(b, dict)) + + +def test_first_turn_never_injects(): + inj = HandoffNoteInjector() + req = _anthropic() + # No prior tier recorded → no transition → no note. + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is False + assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) + + +def test_escalation_transition_injects_note_as_trailing_block(): + inj = HandoffNoteInjector() + # Establish the session on the weak tier first. + inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") + # Now escalate on a real signal → note injected. + req = _anthropic(turn="b") + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True + content = req.body["messages"][-1]["content"] + # tool_result stays first; the note is appended after it as a text block. + assert content[0]["type"] == "tool_result" + assert content[-1] == {"type": "text", "text": DEFAULT_ESCALATION_NOTE} + + +def test_staying_on_a_tier_injects_nothing(): + inj = HandoffNoteInjector() + inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") + req = _anthropic(turn="b") + # Same tier again → no transition → no note (keeps the cache prefix clean). + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is False + assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) + + +def test_escalation_from_ambiguous_source_gated_off_by_default(): + inj = HandoffNoteInjector() + inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") + req = _anthropic(turn="b") + # capable_first can escalate on fall_open (ambiguous default) — the truthful + # gate suppresses the "prior model stalled" note there. + assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is False + + +def test_wrong_signal_gate_can_be_disabled(): + inj = HandoffNoteInjector(only_on_wrong_signal_escalation=False) + inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") + req = _anthropic(turn="b") + assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is True + + +def test_override_source_counts_as_wrong_signal(): + """Compaction / critical-severity escalations stamp ``override`` → note fires.""" + inj = HandoffNoteInjector() + inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") + req = _anthropic(turn="b") + assert inj.maybe_inject(req, tier=CAPABLE, source="override") is True + + +def test_deescalation_off_by_default(): + inj = HandoffNoteInjector() + inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") + req = _anthropic(turn="b") + # No de-escalation note configured → hand-back injects nothing. + assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is False + + +def test_deescalation_note_injected_when_configured(): + inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) + inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") + req = _anthropic(turn="b") + assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is True + assert DEFAULT_DEESCALATION_NOTE in _last_user_text(req) + + +def test_notes_do_not_accumulate_across_turns(): + """Each injected request holds at most one note — nothing carries over.""" + inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) + inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") + up = _anthropic(turn="b") + inj.maybe_inject(up, tier=CAPABLE, source="dimensions") # escalate + down = _anthropic(turn="c") + inj.maybe_inject(down, tier=EFFICIENT, source="dimensions") # de-escalate + # The de-escalation request carries only the de-escalation note, not the + # escalation one (notes live in a single request, never persisted). + text = _last_user_text(down) + assert DEFAULT_DEESCALATION_NOTE in text + assert DEFAULT_ESCALATION_NOTE not in text + + +def test_openai_chat_appends_trailing_user_message(): + inj = HandoffNoteInjector() + first = ChatRequest.openai_chat({"messages": [{"role": "user", "content": "task"}]}) + inj.maybe_inject(first, tier=EFFICIENT, source="fall_open") + req = ChatRequest.openai_chat({ + "messages": [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "", "tool_calls": [ + {"id": "t1", "type": "function", + "function": {"name": "Bash", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "t1", "content": "err"}, + ], + }) + assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True + last = req.body["messages"][-1] + assert last["role"] == "user" + assert last["content"] == DEFAULT_ESCALATION_NOTE From 5b78ff56d7b9e82b02432aafc360179ddd07b6aa Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Mon, 20 Jul 2026 15:27:51 -0700 Subject: [PATCH 09/21] fix(stage_router): drop repeated_cmd_ratio from scorer (0/71 escalations were real loops; spurious Opus bias) Signed-off-by: Sabhatina Selvam --- switchyard/lib/processors/stage_router/scorer.py | 10 +++++++++- tests/test_stage_router_scorer.py | 12 +++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py index 35dbfd21..13281ec8 100644 --- a/switchyard/lib/processors/stage_router/scorer.py +++ b/switchyard/lib/processors/stage_router/scorer.py @@ -48,7 +48,15 @@ #: across the whole stretch — the "latch" that sustains escalation on hard debugging #: tasks (verified against runs where a struggling agent reads-without-writing for #: dozens of turns; treating it as neutral routed those to the weak tier and lost them). -_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring", "repeated_cmd_ratio") +#: +#: ``repeated_cmd_ratio`` was dropped here: an ablation over 178 task-runs (two +#: Nemotron deployments) found 0/71 of its escalations were real command loops — its +#: ``max_count / window`` normalisation makes a single Bash call read as ratio 0.20, +#: so it fired a low-grade CAPABLE bias on any turn that touched the shell, adding +#: ~8% Opus with no accuracy payoff. The Rust signal still computes it for +#: observability; it just no longer influences routing. Re-add with a ``max_count >= 3`` +#: gate if a genuine same-command-loop detector is wanted. +_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring") #: "Making progress" → EFFICIENT (weak). De-escalation is driven by real production #: (writes/edits), not by a clean-result streak: a streak of clean results actively #: cancelled the (windowed) error signal on the same axis and released escalations too diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py index 46ed8107..3b721145 100644 --- a/tests/test_stage_router_scorer.py +++ b/tests/test_stage_router_scorer.py @@ -72,12 +72,14 @@ def test_exploring_is_a_full_escalation_signal(): assert explore.confidence > 0.30 # escalates alone at a typical threshold -def test_repeated_cmd_ratio_is_a_wrong_signal(): - """A weak model looping one command (repeated_cmd_ratio→1) is a full WRONG signal - → CAPABLE, same weight as spinning. Catches churn that severity/spinning miss.""" +def test_repeated_cmd_ratio_does_not_influence_routing(): + """repeated_cmd_ratio was dropped from the WRONG signals (ablation: 0/71 of its + escalations were real loops — it fired on any single shell command). It is still + carried on the dimensions for observability but must contribute nothing to score.""" churn = score(_with(repeated_cmd_ratio=1.0)) - assert churn.score > 0 - assert churn.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) + assert churn.score == 0.0 + assert churn.confidence == 0.0 + assert "repeated_cmd_ratio" not in churn.contributions def test_corroboration_raises_confidence(): From 08e33c85615259613eeb221fa878f161a02b95c9 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 24 Jul 2026 11:20:33 -0700 Subject: [PATCH 10/21] feat(stage_router): stateless handoff notes + strong_system_prompt injection Signed-off-by: Sabhatina Selvam --- ...ge-router-nemu-opus-ef-t50-w5-handoff.yaml | 32 +++++ switchyard/cli/route_bundle.py | 1 + .../processors/stage_router/handoff_notes.py | 115 +++--------------- .../stage_router_request_processor.py | 35 ++++++ switchyard/lib/profiles/stage_router.py | 2 +- .../lib/profiles/stage_router_config.py | 19 +-- 6 files changed, 99 insertions(+), 105 deletions(-) create mode 100644 benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml diff --git a/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml b/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml new file mode 100644 index 00000000..2d6ef95c --- /dev/null +++ b/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml @@ -0,0 +1,32 @@ +# Stage-router ef · nemotron-3-ultra weak · Opus 4.8 @ high · window 5 · t0.50 · handoff notes ON. +# Higher threshold vs tb2-1-stage-router-nemu-opus-ef-t30-w5-handoff — tighter escalation gate. +routes: + switchyard: + type: stage_router + picker: efficient_first + confidence_threshold: 0.50 + signal_recent_window: 5 + fallback_target_on_evict: strong + enable_stats: true + handoff_notes: + enabled: true + only_on_wrong_signal_escalation: true + strong_system_prompt: "Be concise. Prefer short, targeted tool calls over lengthy explanations. Do not re-examine what is already working — focus only on the immediate blocker. Avoid restating context the conversation already contains." + strong: + model: azure/anthropic/claude-opus-4-8 + api_key: ${OPENAI_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: anthropic + timeout_secs: 600 + extra_body: + output_config: + effort: high + weak: + model: nvidia/nvidia/nemotron-3-ultra + api_key: ${OPENAI_API_KEY} + base_url: https://inference-api.nvidia.com/v1 + format: openai + timeout_secs: 600 + extra_body: + chat_template_kwargs: + enable_thinking: false diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 21d6a7eb..0fdead9f 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -241,6 +241,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "signal_recent_window", "classifier", "handoff_notes", + "strong_system_prompt", "enable_stats", "fallback_target_on_evict", }) diff --git a/switchyard/lib/processors/stage_router/handoff_notes.py b/switchyard/lib/processors/stage_router/handoff_notes.py index 7bb8ccdf..66f78b02 100644 --- a/switchyard/lib/processors/stage_router/handoff_notes.py +++ b/switchyard/lib/processors/stage_router/handoff_notes.py @@ -1,28 +1,26 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Optional tier-transition handoff notes. +"""Optional signal-driven handoff notes for the capable (strong) tier. -When the stage_router flips a session between the efficient (weak) and capable -(strong) tiers, this injector appends a short, ephemeral guidance note to the -outgoing request so the model taking over knows *why* it was handed the task. +When the stage_router picks the capable tier because a real signal fired, this +injector appends a short, ephemeral guidance note to the outgoing request so the +model knows *why* it was handed the task. + +**Stateless.** No per-session state is tracked. The note fires on every turn +where the capable tier is picked *and* the decision source was a real signal +(``dimensions`` or ``override``). This avoids the stale-state and hash-collision +problems that came with tracking previous tier per session across a shared proxy. **Ephemeral by construction.** The note is appended to the request the proxy forwards for this turn only; it is never written back into the agent's conversation store. On the next turn the agent re-sends its own history (which -never contained the note) and the injector decides afresh. So notes never -accumulate across turns — a task that escalates, de-escalates, then escalates -again gets at most one note per *transition*, each living in a single request. - -**Transition-gated.** A note is injected only when the picked tier *differs* -from the tier this session used on its previous turn (tracked per session in -:attr:`_last_tier`). Staying on a tier — however many turns — injects nothing, -which keeps the note out of the prompt-cache prefix on the steady-state turns. +never contained the note) and the injector decides afresh. **Truthful escalation gate.** With ``only_on_wrong_signal_escalation`` (default), -the escalation note fires only when the escalation was driven by a real signal +the note fires only when the escalation was driven by a real signal (``override`` — critical severity or compaction — or ``dimensions`` — the scorer -crossing the threshold on WRONG signals), not by an ambiguous default +crossing threshold on wrong signals), not by an ambiguous default (``fall_open`` / ``llm-classifier``). This keeps "the previous model was stalling" from being asserted when no such signal was seen. """ @@ -32,7 +30,7 @@ import logging from typing import TYPE_CHECKING -from switchyard.lib.processors.stage_router.picker import CAPABLE, EFFICIENT +from switchyard.lib.processors.stage_router.picker import CAPABLE if TYPE_CHECKING: from switchyard_rust.core import ChatRequest @@ -60,32 +58,21 @@ #: ambiguous default). Used by the escalation gate. _WRONG_ESCALATION_SOURCES: frozenset[str] = frozenset({"override", "dimensions"}) -#: Cap on the per-session last-tier map so a long-lived proxy process doesn't -#: grow it without bound. When exceeded the whole map is cleared (cheap, and the -#: only cost is a possible missed note on the first turn of in-flight sessions). -_MAX_SESSIONS: int = 8192 - - class HandoffNoteInjector: - """Appends a tier-transition guidance note to the outgoing request. + """Appends a guidance note to capable-tier requests driven by real signals. - One instance per stage_router processor; safe under the single-threaded - asyncio loop (the per-session map is only touched between awaits). + Stateless — no per-session tracking. Fires on every turn where the capable + tier is picked and the decision source was a real wrong signal. """ def __init__( self, *, escalation_note: str = DEFAULT_ESCALATION_NOTE, - deescalation_note: str | None = None, only_on_wrong_signal_escalation: bool = True, - max_sessions: int = _MAX_SESSIONS, ) -> None: self._escalation_note = escalation_note - self._deescalation_note = deescalation_note self._only_on_wrong_signal = only_on_wrong_signal_escalation - self._max_sessions = max_sessions - self._last_tier: dict[str, int] = {} def maybe_inject( self, @@ -94,83 +81,21 @@ def maybe_inject( tier: int, source: str | None, ) -> bool: - """Inject a note if this turn crosses a tier boundary for its session. + """Inject a note when the capable tier is picked due to a real signal. Returns ``True`` iff a note was appended to ``request``. Never raises — any failure degrades to "no note" so it can't block routing. """ try: - key = _session_key(request) - if key is None: + if tier != CAPABLE: return False - prev = self._last_tier.get(key) - self._remember(key, tier) - if prev is None or prev == tier: - return False # first turn of the session, or no transition - - note = self._note_for_transition(prev, tier, source) - if not note: + if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: return False - return _append_note(request, note) + return _append_note(request, self._escalation_note) except Exception: log.debug("handoff-note injection failed; continuing without note", exc_info=True) return False - def _note_for_transition(self, prev: int, tier: int, source: str | None) -> str | None: - if prev == EFFICIENT and tier == CAPABLE: - if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: - return None - return self._escalation_note - if prev == CAPABLE and tier == EFFICIENT: - return self._deescalation_note - return None - - def _remember(self, key: str, tier: int) -> None: - if len(self._last_tier) >= self._max_sessions and key not in self._last_tier: - self._last_tier.clear() - self._last_tier[key] = tier - - -def _session_key(request: ChatRequest) -> str | None: - """Stable per-session fingerprint from the conversation's fixed prefix. - - Claude Code (and agentic callers generally) keep the system prompt and the - first user turn — the task instruction — constant for the life of a session - while appending turns, so a hash of that prefix identifies the session - without relying on an optional session-id header. - """ - body = getattr(request, "body", None) - if not isinstance(body, dict): - return None - parts: list[str] = [] - system = body.get("system") - if isinstance(system, str): - parts.append(system) - elif isinstance(system, list): - parts.append(_text_of(system)) - messages = body.get("messages") - if isinstance(messages, list) and messages: - first = messages[0] - if isinstance(first, dict): - parts.append(_text_of(first.get("content"))) - if not any(parts): - return None - return str(hash("\x1f".join(parts))) - - -def _text_of(content: object) -> str: - """Flatten message/system content (str or list of blocks) to plain text.""" - if isinstance(content, str): - return content - if isinstance(content, list): - out: list[str] = [] - for block in content: - if isinstance(block, dict): - text = block.get("text") - if isinstance(text, str): - out.append(text) - return "".join(out) - return "" def _append_note(request: ChatRequest, note: str) -> bool: diff --git a/switchyard/lib/processors/stage_router_request_processor.py b/switchyard/lib/processors/stage_router_request_processor.py index ba2b396d..ee04596b 100644 --- a/switchyard/lib/processors/stage_router_request_processor.py +++ b/switchyard/lib/processors/stage_router_request_processor.py @@ -50,6 +50,7 @@ def __init__( classifier: TierClassifier | None = None, decision_log: StageRouterDecisionLog | None = None, handoff_injector: HandoffNoteInjector | None = None, + strong_system_prompt: str | None = None, ) -> None: if len(targets) != 2: raise ValueError(f"stage_router requires exactly 2 targets, got {len(targets)}") @@ -60,6 +61,7 @@ def __init__( self._max_index = len(targets) - 1 self._decision_log = decision_log if decision_log is not None else StageRouterDecisionLog() self._handoff_injector = handoff_injector + self._strong_system_prompt = strong_system_prompt self._stats_accumulator: StatsAccumulator | None = None def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None: @@ -93,6 +95,8 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: tier=idx, source=source if isinstance(source, str) else None, ) + if self._strong_system_prompt is not None and idx == CAPABLE: + _prepend_system_prompt(request, self._strong_system_prompt) log.debug( "stage_router pick: idx=%d target=%s model=%s", idx, ctx.selected_target, ctx.selected_model, @@ -123,6 +127,37 @@ async def _record_decision_source(self, ctx: ProxyContext) -> None: log.debug("failed to record stage_router decision source", exc_info=True) +def _prepend_system_prompt(request: ChatRequest, prompt: str) -> None: + """Prepend ``prompt`` to the request's system content. Never raises.""" + try: + body = getattr(request, "body", None) + if not isinstance(body, dict): + return + fmt = getattr(getattr(request, "request_type", None), "value", None) + if fmt == "anthropic": + # Anthropic: top-level "system" field (string or content-block list) + existing = body.get("system", "") + if isinstance(existing, str): + body["system"] = prompt + ("\n\n" + existing if existing else "") + elif isinstance(existing, list): + body["system"] = [{"type": "text", "text": prompt}] + existing + else: + body["system"] = prompt + else: + # OpenAI: prepend a system message before existing messages + messages = body.get("messages") + if not isinstance(messages, list): + return + if messages and messages[0].get("role") == "system": + existing = messages[0].get("content", "") + messages[0] = {"role": "system", "content": prompt + "\n\n" + existing} + else: + messages.insert(0, {"role": "system", "content": prompt}) + request.replace_body(body) + except Exception: + log.debug("strong_system_prompt injection failed; continuing without", exc_info=True) + + __all__ = [ "BUILTIN_PICKERS", "CAPABLE", diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py index 35a7fa43..8bd2533b 100644 --- a/switchyard/lib/profiles/stage_router.py +++ b/switchyard/lib/profiles/stage_router.py @@ -56,6 +56,7 @@ def build(self) -> ComponentChainProfile: classifier=classifier, decision_log=decision_log, handoff_injector=_build_handoff_injector(config.handoff_notes), + strong_system_prompt=config.strong_system_prompt, ) ) @@ -92,7 +93,6 @@ def _build_handoff_injector(config: HandoffNoteConfig | None) -> HandoffNoteInje return None return HandoffNoteInjector( escalation_note=config.escalation_note, - deescalation_note=config.deescalation_note, only_on_wrong_signal_escalation=config.only_on_wrong_signal_escalation, ) diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index 7a92a5c8..a1bce193 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -49,17 +49,13 @@ class HandoffNoteConfig(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - #: Master switch. When ``False`` (default) no note is ever injected, even if - #: the note texts below are customised. + #: Master switch. When ``False`` (default) no note is ever injected. enabled: bool = False - #: Weak→strong note. Injected when a session escalates to the capable tier. + #: Note injected on every capable-tier turn driven by a real wrong signal. escalation_note: str = DEFAULT_ESCALATION_NOTE - #: Strong→weak note. ``None`` (default) disables the de-escalation direction; - #: set a string to also annotate hand-backs to the efficient tier. - deescalation_note: str | None = None - #: When ``True`` (default) only inject the escalation note when the escalation - #: was driven by a real signal (critical severity / compaction override, or the - #: scorer crossing the threshold), not by an ambiguous low-confidence default. + #: When ``True`` (default) only inject when the decision source was a real + #: signal (``dimensions`` or ``override``), not an ambiguous default + #: (``fall_open``). Keeps "stalling" from being asserted without evidence. only_on_wrong_signal_escalation: bool = True @@ -89,6 +85,11 @@ class StageRouterConfig(BaseModel): classifier: ClassifierConfig | None = None #: Optional tier-transition handoff notes (off unless ``handoff_notes.enabled``). handoff_notes: HandoffNoteConfig | None = None + #: Optional system prompt prepended to every request routed to the capable + #: (strong) tier. Injected before the model sees the request — useful for + #: conciseness guidance that should apply on every Opus turn without + #: polluting the agent's conversation history. + strong_system_prompt: str | None = None enable_stats: bool = True @field_validator("capable", "efficient", mode="before") From d6bcb588239619ba5e2a95658a0eaece28377bec Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 24 Jul 2026 11:20:33 -0700 Subject: [PATCH 11/21] feat(stage_router): add weak_system_prompt injection for efficient tier Signed-off-by: Sabhatina Selvam --- .../tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml | 1 + switchyard/cli/route_bundle.py | 1 + switchyard/lib/processors/stage_router_request_processor.py | 4 ++++ switchyard/lib/profiles/stage_router.py | 1 + switchyard/lib/profiles/stage_router_config.py | 3 +++ 5 files changed, 10 insertions(+) diff --git a/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml b/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml index 2d6ef95c..79b46e1f 100644 --- a/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml +++ b/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml @@ -12,6 +12,7 @@ routes: enabled: true only_on_wrong_signal_escalation: true strong_system_prompt: "Be concise. Prefer short, targeted tool calls over lengthy explanations. Do not re-examine what is already working — focus only on the immediate blocker. Avoid restating context the conversation already contains." + weak_system_prompt: "Be concise and efficient. One focused tool call per turn. If the same approach has failed twice, stop and clearly report what failed rather than retrying. Do not explain what you are about to do — just do it." strong: model: azure/anthropic/claude-opus-4-8 api_key: ${OPENAI_API_KEY} diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 0fdead9f..02787f99 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -242,6 +242,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "classifier", "handoff_notes", "strong_system_prompt", + "weak_system_prompt", "enable_stats", "fallback_target_on_evict", }) diff --git a/switchyard/lib/processors/stage_router_request_processor.py b/switchyard/lib/processors/stage_router_request_processor.py index ee04596b..1edc0fa7 100644 --- a/switchyard/lib/processors/stage_router_request_processor.py +++ b/switchyard/lib/processors/stage_router_request_processor.py @@ -51,6 +51,7 @@ def __init__( decision_log: StageRouterDecisionLog | None = None, handoff_injector: HandoffNoteInjector | None = None, strong_system_prompt: str | None = None, + weak_system_prompt: str | None = None, ) -> None: if len(targets) != 2: raise ValueError(f"stage_router requires exactly 2 targets, got {len(targets)}") @@ -62,6 +63,7 @@ def __init__( self._decision_log = decision_log if decision_log is not None else StageRouterDecisionLog() self._handoff_injector = handoff_injector self._strong_system_prompt = strong_system_prompt + self._weak_system_prompt = weak_system_prompt self._stats_accumulator: StatsAccumulator | None = None def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None: @@ -97,6 +99,8 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: ) if self._strong_system_prompt is not None and idx == CAPABLE: _prepend_system_prompt(request, self._strong_system_prompt) + if self._weak_system_prompt is not None and idx == EFFICIENT: + _prepend_system_prompt(request, self._weak_system_prompt) log.debug( "stage_router pick: idx=%d target=%s model=%s", idx, ctx.selected_target, ctx.selected_model, diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py index 8bd2533b..6b7948d0 100644 --- a/switchyard/lib/profiles/stage_router.py +++ b/switchyard/lib/profiles/stage_router.py @@ -57,6 +57,7 @@ def build(self) -> ComponentChainProfile: decision_log=decision_log, handoff_injector=_build_handoff_injector(config.handoff_notes), strong_system_prompt=config.strong_system_prompt, + weak_system_prompt=config.weak_system_prompt, ) ) diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index a1bce193..c5aacf7c 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -90,6 +90,9 @@ class StageRouterConfig(BaseModel): #: conciseness guidance that should apply on every Opus turn without #: polluting the agent's conversation history. strong_system_prompt: str | None = None + #: Optional system prompt prepended to every request routed to the efficient + #: (weak) tier. Useful for conciseness/bail-early guidance on Nemotron turns. + weak_system_prompt: str | None = None enable_stats: bool = True @field_validator("capable", "efficient", mode="before") From cee841999076dc885f39462f040cfb8f6ceb26fe Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 22 Jul 2026 16:25:22 -0700 Subject: [PATCH 12/21] style(stage-router): rustfmt tool_signals test assertions Signed-off-by: Sabhatina Selvam --- .../src/dimension_collector/tool_signals.rs | 69 ++++++++++++------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index c3933514..b00a69c0 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -260,7 +260,10 @@ pub fn extract_tool_signals_with_window( .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - (extract_from_messages_anthropic(messages, recent_window), messages) + ( + extract_from_messages_anthropic(messages, recent_window), + messages, + ) } ChatRequestType::OpenAiResponses => { let items = obj @@ -276,7 +279,10 @@ pub fn extract_tool_signals_with_window( .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - (extract_from_messages_openai_chat(messages, recent_window), messages) + ( + extract_from_messages_openai_chat(messages, recent_window), + messages, + ) } }; @@ -807,16 +813,18 @@ mod tests { #[test] fn tests_passed_detects_pytest_output() { - assert!(detect_tests_passed(&[ - "====== 5 passed in 0.12s ======".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(detect_tests_passed( + &["====== 5 passed in 0.12s ======".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_ignores_partial_failures() { - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] @@ -949,7 +957,10 @@ mod tests { {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, ] })); - assert_eq!(extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, 1.0); + assert_eq!( + extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, + 1.0 + ); } #[test] @@ -987,7 +998,10 @@ mod tests { {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls e\"}"}}]}, ] })); - assert_eq!(extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, 0.2); + assert_eq!( + extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, + 0.2 + ); } #[test] @@ -1052,42 +1066,47 @@ mod tests { #[test] fn tests_passed_detects_pytest_with_failure_block() { // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_cargo_clean_summary() { // Cargo's clean-run summary contains "0 failed" — must not trip the // failure list (regression: previously substring-matched "failed"). - assert!(detect_tests_passed(&[ - "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(detect_tests_passed( + &["running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_rejects_cargo_real_failure() { // Cargo's actual-failure summary: nonzero count before "failed". - assert!(!detect_tests_passed(&[ - "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(!detect_tests_passed( + &["running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_go_clean_summary() { // Go test's clean-run "0 errors" must not trip (regression). - assert!(detect_tests_passed(&[ - "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(detect_tests_passed( + &["ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_pytest_zero_errors() { // Pytest long-form: "0 errors in 0.3s" on a clean run. - assert!(detect_tests_passed(&[ - "5 passed, 0 errors in 0.30s".to_string() - ], DEFAULT_RECENT_WINDOW)); + assert!(detect_tests_passed( + &["5 passed, 0 errors in 0.30s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] From 5493bbcbe3b3b51952586939ca1b1f21ed0d8cf4 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 22 Jul 2026 16:25:22 -0700 Subject: [PATCH 13/21] fix(stage-router): restore configurable stateless de-escalation note Signed-off-by: Sabhatina Selvam --- .../processors/stage_router/handoff_notes.py | 33 +++++++---- switchyard/lib/profiles/stage_router.py | 1 + .../lib/profiles/stage_router_config.py | 3 + tests/test_stage_router_handoff_notes.py | 55 ++++++------------- 4 files changed, 42 insertions(+), 50 deletions(-) diff --git a/switchyard/lib/processors/stage_router/handoff_notes.py b/switchyard/lib/processors/stage_router/handoff_notes.py index 66f78b02..375bc997 100644 --- a/switchyard/lib/processors/stage_router/handoff_notes.py +++ b/switchyard/lib/processors/stage_router/handoff_notes.py @@ -30,7 +30,7 @@ import logging from typing import TYPE_CHECKING -from switchyard.lib.processors.stage_router.picker import CAPABLE +from switchyard.lib.processors.stage_router.picker import CAPABLE, EFFICIENT if TYPE_CHECKING: from switchyard_rust.core import ChatRequest @@ -59,19 +59,23 @@ _WRONG_ESCALATION_SOURCES: frozenset[str] = frozenset({"override", "dimensions"}) class HandoffNoteInjector: - """Appends a guidance note to capable-tier requests driven by real signals. + """Appends a guidance note to a tier-picked request driven by real signals. - Stateless — no per-session tracking. Fires on every turn where the capable - tier is picked and the decision source was a real wrong signal. + Stateless — no per-session tracking. On the capable tier the note fires on + every turn where the decision source was a real wrong signal; on the + efficient tier a de-escalation note fires only when one is configured + (``deescalation_note``), which is off by default. """ def __init__( self, *, escalation_note: str = DEFAULT_ESCALATION_NOTE, + deescalation_note: str | None = None, only_on_wrong_signal_escalation: bool = True, ) -> None: self._escalation_note = escalation_note + self._deescalation_note = deescalation_note self._only_on_wrong_signal = only_on_wrong_signal_escalation def maybe_inject( @@ -81,17 +85,22 @@ def maybe_inject( tier: int, source: str | None, ) -> bool: - """Inject a note when the capable tier is picked due to a real signal. + """Inject the guidance note appropriate to the picked tier. - Returns ``True`` iff a note was appended to ``request``. Never raises — - any failure degrades to "no note" so it can't block routing. + Capable tier: the escalation note, gated to real wrong signals when + ``only_on_wrong_signal_escalation``. Efficient tier: the de-escalation + note, only when one is configured. Returns ``True`` iff a note was + appended. Never raises — any failure degrades to "no note" so it can't + block routing. """ try: - if tier != CAPABLE: - return False - if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: - return False - return _append_note(request, self._escalation_note) + if tier == CAPABLE: + if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES: + return False + return _append_note(request, self._escalation_note) + if tier == EFFICIENT and self._deescalation_note is not None: + return _append_note(request, self._deescalation_note) + return False except Exception: log.debug("handoff-note injection failed; continuing without note", exc_info=True) return False diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py index 6b7948d0..32a0b4e9 100644 --- a/switchyard/lib/profiles/stage_router.py +++ b/switchyard/lib/profiles/stage_router.py @@ -94,6 +94,7 @@ def _build_handoff_injector(config: HandoffNoteConfig | None) -> HandoffNoteInje return None return HandoffNoteInjector( escalation_note=config.escalation_note, + deescalation_note=config.deescalation_note, only_on_wrong_signal_escalation=config.only_on_wrong_signal_escalation, ) diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index c5aacf7c..0092bb59 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -53,6 +53,9 @@ class HandoffNoteConfig(BaseModel): enabled: bool = False #: Note injected on every capable-tier turn driven by a real wrong signal. escalation_note: str = DEFAULT_ESCALATION_NOTE + #: Optional strong→weak note. When set, injected on every efficient-tier turn + #: (hand-back to the weak model); ``None`` (default) leaves de-escalation off. + deescalation_note: str | None = None #: When ``True`` (default) only inject when the decision source was a real #: signal (``dimensions`` or ``override``), not an ambiguous default #: (``fall_open``). Keeps "stalling" from being asserted without evidence. diff --git a/tests/test_stage_router_handoff_notes.py b/tests/test_stage_router_handoff_notes.py index 8f84d8ad..7afc4b97 100644 --- a/tests/test_stage_router_handoff_notes.py +++ b/tests/test_stage_router_handoff_notes.py @@ -1,7 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for the optional tier-transition handoff-note injector.""" +"""Unit tests for the optional handoff-note injector. + +The injector is stateless: on the capable tier it appends an escalation note +when the decision source is a real wrong signal; on the efficient tier it +appends a de-escalation note only when one is configured. +""" from __future__ import annotations @@ -35,20 +40,9 @@ def _last_user_text(request: ChatRequest) -> str: return "".join(b.get("text", "") for b in content if isinstance(b, dict)) -def test_first_turn_never_injects(): +def test_escalation_injects_note_as_trailing_block(): inj = HandoffNoteInjector() req = _anthropic() - # No prior tier recorded → no transition → no note. - assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is False - assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) - - -def test_escalation_transition_injects_note_as_trailing_block(): - inj = HandoffNoteInjector() - # Establish the session on the weak tier first. - inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") - # Now escalate on a real signal → note injected. - req = _anthropic(turn="b") assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True content = req.body["messages"][-1]["content"] # tool_result stays first; the note is appended after it as a text block. @@ -56,51 +50,39 @@ def test_escalation_transition_injects_note_as_trailing_block(): assert content[-1] == {"type": "text", "text": DEFAULT_ESCALATION_NOTE} -def test_staying_on_a_tier_injects_nothing(): - inj = HandoffNoteInjector() - inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") - req = _anthropic(turn="b") - # Same tier again → no transition → no note (keeps the cache prefix clean). - assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is False - assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) - - def test_escalation_from_ambiguous_source_gated_off_by_default(): inj = HandoffNoteInjector() - inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") - req = _anthropic(turn="b") + req = _anthropic() # capable_first can escalate on fall_open (ambiguous default) — the truthful # gate suppresses the "prior model stalled" note there. assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is False + assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req) def test_wrong_signal_gate_can_be_disabled(): inj = HandoffNoteInjector(only_on_wrong_signal_escalation=False) - inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") - req = _anthropic(turn="b") + req = _anthropic() assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is True def test_override_source_counts_as_wrong_signal(): """Compaction / critical-severity escalations stamp ``override`` → note fires.""" inj = HandoffNoteInjector() - inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") - req = _anthropic(turn="b") + req = _anthropic() assert inj.maybe_inject(req, tier=CAPABLE, source="override") is True def test_deescalation_off_by_default(): inj = HandoffNoteInjector() - inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") - req = _anthropic(turn="b") - # No de-escalation note configured → hand-back injects nothing. + req = _anthropic() + # No de-escalation note configured → hand-back to the weak tier injects nothing. assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is False + assert DEFAULT_DEESCALATION_NOTE not in _last_user_text(req) def test_deescalation_note_injected_when_configured(): inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) - inj.maybe_inject(_anthropic(turn="a"), tier=CAPABLE, source="dimensions") - req = _anthropic(turn="b") + req = _anthropic() assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is True assert DEFAULT_DEESCALATION_NOTE in _last_user_text(req) @@ -108,13 +90,12 @@ def test_deescalation_note_injected_when_configured(): def test_notes_do_not_accumulate_across_turns(): """Each injected request holds at most one note — nothing carries over.""" inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE) - inj.maybe_inject(_anthropic(turn="a"), tier=EFFICIENT, source="fall_open") up = _anthropic(turn="b") inj.maybe_inject(up, tier=CAPABLE, source="dimensions") # escalate down = _anthropic(turn="c") inj.maybe_inject(down, tier=EFFICIENT, source="dimensions") # de-escalate - # The de-escalation request carries only the de-escalation note, not the - # escalation one (notes live in a single request, never persisted). + # Each request is built fresh, so the de-escalation request carries only the + # de-escalation note, never the escalation one. text = _last_user_text(down) assert DEFAULT_DEESCALATION_NOTE in text assert DEFAULT_ESCALATION_NOTE not in text @@ -122,8 +103,6 @@ def test_notes_do_not_accumulate_across_turns(): def test_openai_chat_appends_trailing_user_message(): inj = HandoffNoteInjector() - first = ChatRequest.openai_chat({"messages": [{"role": "user", "content": "task"}]}) - inj.maybe_inject(first, tier=EFFICIENT, source="fall_open") req = ChatRequest.openai_chat({ "messages": [ {"role": "user", "content": "task"}, From 84534b5ca70bb6d6c869d4f43fed2820cbd342ac Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 22 Jul 2026 17:32:20 -0700 Subject: [PATCH 14/21] docs(stage-router): update signals/calibration for corroborative scorer; replace calibration scripts with scorer skill Signed-off-by: Sabhatina Selvam --- .../switchyard-stage-router-scorer/SKILL.md | 93 ++++++ .../calibration/stage_router/calibrate.py | 146 --------- .../stage_router/signal_extractor.py | 297 ------------------ benchmark/calibration/stage_router/sweep.py | 121 ------- benchmark/score_run.py | 282 +++++++++++++++++ .../stage_router_routing.md | 73 +++-- 6 files changed, 417 insertions(+), 595 deletions(-) create mode 100644 .agents/skills/switchyard-stage-router-scorer/SKILL.md delete mode 100644 benchmark/calibration/stage_router/calibrate.py delete mode 100644 benchmark/calibration/stage_router/signal_extractor.py delete mode 100644 benchmark/calibration/stage_router/sweep.py create mode 100644 benchmark/score_run.py diff --git a/.agents/skills/switchyard-stage-router-scorer/SKILL.md b/.agents/skills/switchyard-stage-router-scorer/SKILL.md new file mode 100644 index 00000000..01e57e6f --- /dev/null +++ b/.agents/skills/switchyard-stage-router-scorer/SKILL.md @@ -0,0 +1,93 @@ +# Skill: switchyard-stage-router-scorer + +**description**: Score benchmark run trajectories through the stage-router Rust scorer and picker, then visualise score distributions. Use when you want to replay trajectories through the picker, analyse routing splits, or compare score distributions across configs. + +## Scripts + +| Script | Purpose | Input | Output | +|--------|---------|-------|--------| +| `benchmark/score_run.py` | Score a live run dir via real picker | run dir path | per-turn JSONL + per-task CSV | + +## Quick Reference + +```bash +# Score a run +uv run python benchmark/score_run.py --run benchmark/tb_runs/ +# → /tmp/-scores.jsonl (per turn) +# → /tmp/-per-task.csv (per task) + +# Custom threshold or window +uv run python benchmark/score_run.py \ + --run benchmark/tb_runs/ \ + --threshold 0.15 --window 3 +``` + +## Scoring pipeline (what score_run.py does per turn) + +``` +trajectory step (tool_use + tool_result) + → append to cumulative Anthropic messages list + → ChatRequest.anthropic({"model": ..., "messages": messages}) # Rust binding + → dc.process(ctx, request) # DimensionCollector — one per task, accumulates state + → get_tool_result_signal(ctx) # read signal from ctx + → from_signal(signal) + scorer_score(dims) # raw score + confidence (for analysis) + → pick_capable_first(ctx, threshold) # actual cf decision (same ctx, no re-process) + → pick_efficient_first(ctx, threshold) # actual ef decision (same ctx, no re-process) +``` + +**Key:** `dc.process()` is called **once per turn** on a single ctx. Both pickers read the signal +already stored in that ctx — no duplicate processing. + +**What the picker does beyond raw score:** +- `_apply_overrides`: `severity >= 1.0` → force CAPABLE; `tests_passed AND depth >= 10 AND writes <= 1` → force EFFICIENT +- `confidence < threshold` → fall_open to default tier (CAPABLE for cf, EFFICIENT for ef) +- Only when `confidence >= threshold`: route by score direction + +## Per-turn JSONL schema + +```json +{ + "task_name": "terminal-bench/...", "trial_name": "...", "run_id": "...", + "reward": 1.0, "turn_depth": 5, + "score": -0.83, "confidence": 0.95, + "tool_name": "Bash", "is_error": false, + "write_count": 2, "edit_count": 1, "read_count": 3, + "no_error_streak": 4, "pure_bash_streak": 2, "tests_passed": false, + "pick_cf": 1, + "pick_ef": 0 +} +``` + +`pick_cf` / `pick_ef`: `1` = CAPABLE (Opus), `0` = EFFICIENT (Nemotron) + +## Per-task CSV columns + +`run_id, task_name, reward, n_turns, mean_score, mean_confidence, +pct_strong_clear, pct_strong_uncertain, pct_weak_uncertain, pct_weak_clear, +opus_pct_cf, nemotron_pct_cf, opus_pct_ef, nemotron_pct_ef` + +`opus_pct_cf` / `opus_pct_ef` are derived from actual `pick_cf` / `pick_ef` decisions, not score bands. + +## Band definitions (for histogram colouring, threshold T) + +| Band | Score range | cf default | ef default | +|------|-------------|------------|------------| +| strong_clear | ≥ T | Opus | Opus | +| strong_uncertain | (0, T) | Opus (fall_open) | Nemotron (fall_open) | +| weak_uncertain | (-T, 0) | Nemotron (fall_open) | Nemotron (fall_open) | +| weak_clear | ≤ -T | Nemotron | Nemotron | + +Overrides can change any band. Use `pick_cf`/`pick_ef` for the true decision. + +## Key findings (v0.2.0 baseline tarballs, T=0.20) + +- Distribution is **highly bimodal** — most turns land in strong_clear or weak_clear +- `cf, t=0.20`: ~34% Opus on partial run; ~42% on full baseline +- `ef, t=0.20`: ~20% Opus on partial run; ~41% on full baseline +- Live routing split differs from shadow score — Nemotron trajectories are shorter and reshape the distribution + +## Anti-patterns + +- Don't call `dc.process()` multiple times per turn for different pickers — both pickers read from the same ctx. One `dc.process()` call per turn is correct. +- Don't infer routing split from score bands alone — overrides and fall_open change the actual decision. Always use `pick_cf` / `pick_ef`. +- Don't compare cost directly across configs: Nemotron has ~39% cache hit rate vs ~92% for Opus. diff --git a/benchmark/calibration/stage_router/calibrate.py b/benchmark/calibration/stage_router/calibrate.py deleted file mode 100644 index 7b8a97d6..00000000 --- a/benchmark/calibration/stage_router/calibrate.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Calibrate stage_router confidence threshold from Harbor run outputs. - -Pass the Harbor run output directories for the pure-capable and pure-efficient arms: - - python calibrate.py --capable-run-dir /tmp/runs/capable --efficient-run-dir /tmp/runs/efficient - python sweep.py -""" -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -from signal_extractor import replay_trajectory, task_summary # noqa: E402 - -OUT = Path(__file__).parent - -_RANK = {"pass": 2, "fail": 1, "err": 0} - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Calibrate stage_router confidence threshold from Harbor run outputs." - ) - parser.add_argument( - "--capable-run-dir", - type=Path, - required=True, - help="Harbor output directory for the pure-capable run.", - ) - parser.add_argument( - "--efficient-run-dir", - type=Path, - required=True, - help="Harbor output directory for the pure-efficient probe run.", - ) - return parser.parse_args(argv) - - -def _outcome(result: dict) -> str: - if result.get("exception_info") is not None: - return "err" - reward = (result.get("verifier_result") or {}).get("rewards", {}).get("reward") - return "pass" if reward == 1.0 else "fail" - - -def read_arm(arm_dir: Path): - if not arm_dir.is_dir(): - raise FileNotFoundError(f"{arm_dir} is not a directory") - - outcomes: dict[str, str] = {} - feats: dict[str, dict] = {} - per_turn: dict[str, list] = {} - - for td in sorted(arm_dir.iterdir()): - if not td.is_dir(): - continue - rp = td / "result.json" - if not rp.exists(): - continue - result = json.loads(rp.read_text()) - task = result.get("task_name") or td.name - outcome = _outcome(result) - trajs = list(td.glob("agent/sessions/projects/-app/*.jsonl")) - signals = replay_trajectory(trajs[0]) if trajs else [] - - if _RANK[outcome] > _RANK.get(outcomes.get(task, ""), -1): - outcomes[task] = outcome - feats[task] = task_summary(signals) - per_turn[task] = [ - { - "task": task, - "turn": i, - "severity": s.severity, - "write_count": s.write_count, - "edit_count": s.edit_count, - "read_count": s.read_count, - "turn_depth": s.turn_depth, - "pure_bash_streak": s.pure_bash_streak, - "tests_passed": s.tests_passed, - } - for i, s in enumerate(signals) - ] - return outcomes, feats, per_turn - - -def main(argv: list[str] | None = None): - args = parse_args(argv) - arms = { - "capable": args.capable_run_dir, - "efficient": args.efficient_run_dir, - } - all_outcomes: dict[str, dict] = {} - all_feats: dict[str, dict] = {} - all_turns: dict[str, dict] = {} - - for arm, d in arms.items(): - outcomes, feats, turns = read_arm(d) - all_outcomes[arm] = outcomes - all_feats[arm] = feats - all_turns[arm] = turns - passes = sum(1 for v in outcomes.values() if v == "pass") - print(f"{arm}: {len(outcomes)} tasks pass={passes}") - - arm_names = list(arms) - if len(arm_names) >= 2: - s_out = all_outcomes[arm_names[0]] - w_out = all_outcomes[arm_names[1]] - buckets: dict[str, int] = {"RESCUE": 0, "LOSS": 0, "SAFE": 0, "HARD": 0} - for t in set(s_out) | set(w_out): - s, w = s_out.get(t), w_out.get(t) - if s == "pass" and w == "pass": - buckets["SAFE"] += 1 - elif s == "fail" and w == "pass": - buckets["RESCUE"] += 1 - elif s == "pass" and w != "pass": - buckets["LOSS"] += 1 - elif s == "fail" and w != "pass": - buckets["HARD"] += 1 - print() - for b, n in buckets.items(): - print(f" {b}: {n}") - - with (OUT / "per_task.jsonl").open("w") as f: - for arm in arm_names: - for task, outcome in all_outcomes[arm].items(): - f.write(json.dumps({"task": task, "arm": arm, "outcome": outcome, - **all_feats[arm].get(task, {})}) + "\n") - - with (OUT / "per_turn.jsonl").open("w") as f: - for arm in arm_names: - for _task, turn_list in all_turns[arm].items(): - for rec in turn_list: - f.write(json.dumps({**rec, "arm": arm}) + "\n") - - print(f"\nWrote per_task.jsonl + per_turn.jsonl → {OUT}") - print("Run sweep.py to score escalation policies.") - - -if __name__ == "__main__": - main() diff --git a/benchmark/calibration/stage_router/signal_extractor.py b/benchmark/calibration/stage_router/signal_extractor.py deleted file mode 100644 index 007d1311..00000000 --- a/benchmark/calibration/stage_router/signal_extractor.py +++ /dev/null @@ -1,297 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python port of crates/switchyard-components/src/dimension_collector/tool_signals.rs. - -Reads a claude-code session JSONL trajectory and reconstructs the per-turn -ToolResultSignal that the stage_router picker would have seen. Output is one record -per assistant turn. -""" -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -SOFT, HARD, CRITICAL = 0.3, 0.7, 1.0 - -ERROR_PATTERNS: list[tuple[str, float, tuple[str, ...]]] = [ - ("oom", CRITICAL, ("out of memory", "memoryerror", "cannot allocate memory")), - ("connection_refused", CRITICAL, ("connection refused", "connectionrefusederror", "econnrefused")), - ("traceback", HARD, ("traceback (most recent call last)",)), - ("import_error", HARD, ("modulenotfounderror:", "importerror:", "no module named ")), - ("cmd_not_found", HARD, ("command not found", "not found\n", "/usr/bin/env: ")), - ("assertion", HARD, ("assertionerror",)), - ("value_error", HARD, ("valueerror:",)), - ("syntax_error", HARD, ("syntaxerror:",)), - ("timeout", HARD, ("timed out", "timeouterror", "timeout expired", "deadline exceeded")), - ("no_such_file", HARD, ("filenotfounderror:", "no such file or directory")), - ("exit_nonzero", SOFT, ("exit code 1", "exit code 2", "exit status 1", "returned non-zero", "exited with code")), -] - -EDIT_TOOL_NAMES = {"edit", "multiedit", "notebookedit", "str_replace", "str_replace_based_edit_tool", "text_editor"} -WRITE_TOOL_NAMES = {"write", "create_file", "new_file"} -READ_TOOL_NAMES = {"read", "view"} -PLAN_TOOL_NAMES = {"todowrite", "todo_write", "todo", "update_plan"} -BASH_TOOL_NAMES = {"bash", "shell_command", "shell", "local_shell_call"} - -BASH_WRITE_PATTERNS = ("cat >", "cat >>", "echo >", "echo >>", "tee ", "printf >", "printf >>", "> /", ">> /", "<< 'eof'", "< tuple[float, list[str]]: - lower = text.lower() - patterns: list[str] = [] - severity = 0.0 - for name, sev, subs in ERROR_PATTERNS: - if any(s in lower for s in subs): - patterns.append(name) - if sev > severity: - severity = sev - return severity, patterns - - -def classify_tool_call(name: str, command: str | None) -> str: - lower = name.lower() - if lower in WRITE_TOOL_NAMES: - return "Write" - if lower in EDIT_TOOL_NAMES: - return "Edit" - if lower in READ_TOOL_NAMES: - return "Read" - if lower in PLAN_TOOL_NAMES: - return "Plan" - if lower in BASH_TOOL_NAMES and command is not None: - if any(p in command for p in BASH_WRITE_PATTERNS): - return "Write" - if any(p in command for p in BASH_EDIT_PATTERNS): - return "Edit" - if any(p in command for p in BASH_READ_PATTERNS): - return "Read" - return "Other" - - -def _detect_tests_passed(tool_texts: list[str]) -> bool: - recent = tool_texts[-3:] if len(tool_texts) > 3 else tool_texts - for text in recent: - lower = text.lower() - if any(p in lower for p in TEST_PASS_PHRASES) and not any(p in lower for p in TEST_FAILURE_PHRASES): - return True - return False - - -def _compute_no_error_streak(tool_texts: list[str]) -> int: - streak = 0 - for text in reversed(tool_texts): - sev, _ = classify_text(text) - if sev > 0.0: - break - streak += 1 - return streak - - -def _build_signal(tool_texts: list[str], tool_calls: list[tuple[str, str | None]], turn_depth: int, prompt_char_count: int) -> ToolResultSignal: - severity, patterns = classify_text(tool_texts[-1]) if tool_texts else (0.0, []) - no_error_streak = _compute_no_error_streak(tool_texts) - - recent_start = max(0, len(tool_calls) - RECENT_WINDOW) - write_count = edit_count = read_count = todowrite_count = 0 - recent_write_count = recent_edit_count = recent_read_count = recent_todowrite_count = 0 - pure_bash_streak = 0 - streak_open = True - - for i in range(len(tool_calls) - 1, -1, -1): - name, cmd = tool_calls[i] - cat = classify_tool_call(name, cmd) - if streak_open: - if cat == "Other": - pure_bash_streak += 1 - else: - streak_open = False - in_recent = i >= recent_start - if cat == "Write": - write_count += 1 - if in_recent: - recent_write_count += 1 - elif cat == "Edit": - edit_count += 1 - if in_recent: - recent_edit_count += 1 - elif cat == "Read": - read_count += 1 - if in_recent: - recent_read_count += 1 - elif cat == "Plan": - todowrite_count += 1 - if in_recent: - recent_todowrite_count += 1 - - return ToolResultSignal( - severity=severity, - patterns=patterns, - no_error_streak=no_error_streak, - edit_count=edit_count, - write_count=write_count, - read_count=read_count, - todowrite_count=todowrite_count, - recent_edit_count=recent_edit_count, - recent_write_count=recent_write_count, - recent_read_count=recent_read_count, - recent_todowrite_count=recent_todowrite_count, - pure_bash_streak=pure_bash_streak, - tests_passed=_detect_tests_passed(tool_texts), - turn_depth=turn_depth, - prompt_char_count=prompt_char_count, - ) - - -def _content_to_text(content: Any) -> str | None: - if isinstance(content, str): - return content - if isinstance(content, list): - parts = [] - for b in content: - if isinstance(b, dict) and b.get("type") == "text" and isinstance(b.get("text"), str): - parts.append(b["text"]) - return "\n".join(parts) if parts else None - return None - - -def replay_trajectory(jsonl_path: Path) -> list[ToolResultSignal]: - """Walk a claude-code session JSONL and emit per-assistant-turn signals. - - Mirrors `extract_from_messages_anthropic` in tool_signals.rs: each assistant - turn produces a signal computed from all messages up to (but not including) - the assistant turn itself — i.e. what the picker would have seen as input - when deciding which model to call for that turn. - """ - tool_texts: list[str] = [] - tool_calls: list[tuple[str, str | None]] = [] - prompt_char_count = 0 - msg_count = 0 # messages-so-far (proxy for turn_depth in the Rust impl) - signals: list[ToolResultSignal] = [] - - with open(jsonl_path, encoding="utf-8", errors="replace") as f: - for line in f: - try: - ev = json.loads(line) - except json.JSONDecodeError: - continue - et = ev.get("type") - if et not in ("user", "assistant"): - continue - - if et == "assistant": - # Snapshot signal before processing the assistant content — this - # is what the picker would have seen when deciding which model - # serves THIS assistant turn. - signals.append(_build_signal(tool_texts.copy(), tool_calls.copy(), msg_count, prompt_char_count)) - - msg = ev.get("message", {}) - content = msg.get("content") - msg_count += 1 - - if et == "user": - # Claude-code stores user content as either a plain string - # (initial prompt or text follow-up) or a list with - # tool_result / text blocks. - if isinstance(content, str): - prompt_char_count = len(content) - elif isinstance(content, list): - for b in content: - if not isinstance(b, dict): - continue - bt = b.get("type") - if bt == "tool_result": - text = _content_to_text(b.get("content")) - if text is not None: - tool_texts.append(text) - elif bt == "text": - t = b.get("text") - if isinstance(t, str): - prompt_char_count = len(t) - elif et == "assistant": - if isinstance(content, list): - for b in content: - if not isinstance(b, dict): - continue - if b.get("type") == "tool_use": - name = b.get("name") - inp = b.get("input") - cmd = None - if isinstance(inp, dict): - c = inp.get("command") - if isinstance(c, str): - cmd = c.lower() - if isinstance(name, str): - tool_calls.append((name, cmd)) - - return signals - - -def task_summary(signals: list[ToolResultSignal]) -> dict[str, Any]: - """Aggregate per-turn signals into one feature vector per task.""" - if not signals: - return { - "turns": 0, - "max_severity": 0.0, - "max_turn_depth": 0, - "max_pure_bash": 0, - "total_writes": 0, - "total_edits": 0, - "total_reads": 0, - "total_todowrites": 0, - "max_recent_reads": 0, - "max_recent_writes": 0, - "max_recent_todowrites": 0, - "prompt_char_count": 0, - "ever_tests_passed": False, - "ever_severity_critical": False, - "ever_severity_hard": False, - "ever_severity_soft": False, - } - last = signals[-1] - return { - "turns": len(signals), - "max_severity": max(s.severity for s in signals), - "max_turn_depth": max(s.turn_depth for s in signals), - "max_pure_bash": max(s.pure_bash_streak for s in signals), - "total_writes": last.write_count, - "total_edits": last.edit_count, - "total_reads": last.read_count, - "total_todowrites": last.todowrite_count, - "max_recent_reads": max(s.recent_read_count for s in signals), - "max_recent_writes": max(s.recent_write_count for s in signals), - "max_recent_todowrites": max(s.recent_todowrite_count for s in signals), - "prompt_char_count": signals[0].prompt_char_count if signals else 0, - "ever_tests_passed": any(s.tests_passed for s in signals), - "ever_severity_critical": any(s.severity >= CRITICAL for s in signals), - "ever_severity_hard": any(s.severity >= HARD for s in signals), - "ever_severity_soft": any(s.severity >= SOFT for s in signals), - } diff --git a/benchmark/calibration/stage_router/sweep.py b/benchmark/calibration/stage_router/sweep.py deleted file mode 100644 index 11950c21..00000000 --- a/benchmark/calibration/stage_router/sweep.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Score escalation policies against per_turn.jsonl + per_task.jsonl. - -Run calibrate.py first to generate the input files. -""" -from __future__ import annotations - -import json -from collections import defaultdict -from pathlib import Path - -OUT = Path(__file__).parent - - -def load(): - by: dict = defaultdict(list) - with (OUT / "per_turn.jsonl").open() as f: - for line in f: - r = json.loads(line) - by[(r["task"], r["arm"])].append(r) - for k in by: - by[k].sort(key=lambda r: r["turn"]) - - outcomes: dict = defaultdict(dict) - rank = {"pass": 2, "fail": 1, "err": 0} - with (OUT / "per_task.jsonl").open() as f: - for line in f: - r = json.loads(line) - cur = outcomes[r["task"]].get(r["arm"]) - if cur is None or rank[r["outcome"]] > rank[cur]: - outcomes[r["task"]][r["arm"]] = r["outcome"] - return by, outcomes - - -def score(by, outcomes, picker_fn, capable="capable", efficient="efficient"): - P = F = E = esc = 0 - for task, arms in outcomes.items(): - turns = by.get((task, capable), []) - escalate = picker_fn(turns) if turns else False - o = arms.get(efficient if escalate else capable, "err") - if escalate: - esc += 1 - if o == "pass": - P += 1 - elif o == "fail": - F += 1 - else: - E += 1 - total = P + F - return { - "pass_count": P, - "pct": P / max(1, total) * 100, - "esc_rate": esc / max(1, len(outcomes)), - } - - -# --- Escalation policies --- - -def always_stay(turns): return False -def always_escalate(turns): return True - - -def no_write_by_turn(N: int): - def p(turns): - for t in turns: - if t["turn_depth"] >= N: - return t["write_count"] == 0 and t["edit_count"] == 0 - return False - return p - - -def silent_stall(td: int, max_reads: int = 4): - def p(turns): - return any( - t["turn_depth"] >= td and t["write_count"] == 0 - and t["edit_count"] == 0 and t["read_count"] <= max_reads - for t in turns - ) - return p - - -def combo(turns): - """Escalate on stall or critical error; stay if tests passed.""" - for t in turns: - if t["tests_passed"]: - return False - if t["severity"] >= 1.0: - return True - if t["turn_depth"] >= 8 and t["write_count"] == 0 and t["edit_count"] == 0: - return True - if t["turn_depth"] >= 7 and t["severity"] >= 0.7 and t["write_count"] == 0: - return True - if t["pure_bash_streak"] >= 4 and t["write_count"] == 0: - return True - return False - - -def main(): - by, outcomes = load() - policies = [ - ("always_stay", always_stay), - ("always_escalate", always_escalate), - ("no_write_by_turn=8", no_write_by_turn(8)), - ("no_write_by_turn=12", no_write_by_turn(12)), - ("no_write_by_turn=15", no_write_by_turn(15)), - ("silent_stall td>=8 R<=2", silent_stall(8, 2)), - ("silent_stall td>=8 R<=4", silent_stall(8, 4)), - ("silent_stall td>=12 R<=6", silent_stall(12, 6)), - ("combo", combo), - ] - print(f"{'policy':40} pass% esc%") - print("-" * 55) - for name, p in policies: - r = score(by, outcomes, p) - print(f" {name:38} {r['pct']:5.1f}% {r['esc_rate']*100:4.0f}%") - - -if __name__ == "__main__": - main() diff --git a/benchmark/score_run.py b/benchmark/score_run.py new file mode 100644 index 00000000..f34e4189 --- /dev/null +++ b/benchmark/score_run.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Score a benchmark run directory via the stage-router Rust scorer. + +Reads trajectory.json from each completed task, feeds each tool-use turn through: + DimensionCollector.process(ctx, request) → pick_capable_first / pick_efficient_first + +The picker functions replicate live routing exactly: overrides, scorer, and fall_open. + +Usage: + uv run python benchmark/score_run.py --run benchmark/tb_runs/ + uv run python benchmark/score_run.py --run benchmark/tb_runs/ \\ + --output /tmp/scores.jsonl --threshold 0.20 --window 3 +""" +import argparse +import asyncio +import csv +import json +import sys +from collections import defaultdict +from pathlib import Path +from statistics import mean + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from switchyard.lib.processors.stage_router.dimensions import from_signal +from switchyard.lib.processors.stage_router.picker import ( + CAPABLE, + pick_capable_first, + pick_efficient_first, +) +from switchyard.lib.processors.stage_router.scorer import score as scorer_score +from switchyard_rust.components import DimensionCollector, get_tool_result_signal +from switchyard_rust.core import ChatRequest, ProxyContext + +RECENT_WINDOW = 3 + + +def _tool_use_id(message: str) -> str: + parts = message.split() + return parts[-1] if len(parts) >= 2 else f"tu_{id(message)}" + + +async def score_trajectory( + traj: dict, + reward: float | None, + task_name: str, + trial_name: str, + run_id: str, + recent_window: int, + confidence_threshold: float, +) -> list[dict]: + steps = traj.get("steps", []) + + # One DimensionCollector per task — accumulates state across turns + dc = DimensionCollector(recent_window=recent_window) + await dc.startup() + + rows: list[dict] = [] + messages: list[dict] = [] + + for s in steps: + if s["source"] == "user" and not (s.get("extra") or {}).get("is_sidechain"): + messages.append({"role": "user", "content": [{"type": "text", "text": s["message"]}]}) + break + + if not messages: + return rows + + for s in steps: + if s["source"] != "agent": + continue + extra = s.get("extra") or {} + if extra.get("is_sidechain"): + continue + + tool_name = extra.get("tool_use_name") + if not tool_name: + text = s.get("message", "") + if text: + messages.append({"role": "assistant", "content": [{"type": "text", "text": text}]}) + continue + + tool_use_id = _tool_use_id(s.get("message", "")) + raw_args = extra.get("raw_arguments") or {} + is_error = bool(extra.get("tool_result_is_error", False)) + + metadata = extra.get("metadata") or {} + raw_result = metadata.get("raw_tool_result") or {} + content = raw_result.get("content", "") + if not isinstance(content, str): + content = json.dumps(content) + + messages.append({ + "role": "assistant", + "content": [{"type": "tool_use", "id": tool_use_id, "name": tool_name, "input": raw_args}], + }) + messages.append({ + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": content, "is_error": is_error}], + }) + + # Build Anthropic ChatRequest and process once through DimensionCollector + ctx = ProxyContext() + request = ChatRequest.anthropic({ + "model": "claude-opus-4-8", + "max_tokens": 8096, + "messages": messages, + }) + await dc.process(ctx, request) + + signal = get_tool_result_signal(ctx) + if signal is None: + continue + + # Raw score for analysis: wrong signals score positive (→CAPABLE), + # progress signals negative (→EFFICIENT). Picker-independent, so the + # histogram shows the capable/efficient separation directly. + dims = from_signal(signal) + sr = scorer_score(dims) # fixed weights; threshold dials corroboration + + # Actual picker decisions on the same ctx — both just read the signal, + # no extra dc.process() calls needed + tier_cf = await pick_capable_first(ctx, confidence_threshold) + tier_ef = await pick_efficient_first(ctx, confidence_threshold) + + rows.append({ + "task_name": task_name, + "trial_name": trial_name, + "run_id": run_id, + "reward": reward, + "turn_depth": signal.turn_depth, + "score": sr.score, + "confidence": sr.confidence, + "tool_name": tool_name, + "is_error": is_error, + "write_count": signal.write_count, + "edit_count": signal.edit_count, + "read_count": signal.read_count, + "no_error_streak": signal.no_error_streak, + "pure_bash_streak": signal.pure_bash_streak, + "tests_passed": signal.tests_passed, + "pick_cf": tier_cf, # CAPABLE=1, EFFICIENT=0 + "pick_ef": tier_ef, + }) + + return rows + + +def band(score: float, threshold: float) -> str: + if score >= threshold: + return "strong_clear" + if score > 0: + return "strong_uncertain" + if score <= -threshold: + return "weak_clear" + if score < 0: + return "weak_uncertain" + return "zero" + + +def write_per_task_csv(all_rows: list[dict], csv_path: Path, threshold: float) -> None: + groups: dict[tuple, list] = defaultdict(list) + for r in all_rows: + groups[(r["run_id"], r["task_name"])].append(r) + + fieldnames = [ + "run_id", "task_name", "reward", "n_turns", + "mean_score", "mean_confidence", + "pct_strong_clear", "pct_strong_uncertain", "pct_weak_uncertain", "pct_weak_clear", + "opus_pct_cf", "nemotron_pct_cf", "opus_pct_ef", "nemotron_pct_ef", + ] + rows_out = [] + for (run_id, task_name), turns in sorted(groups.items()): + n = len(turns) + scores = [t["score"] for t in turns] + confs = [t["confidence"] for t in turns] + reward = turns[0]["reward"] + bands = [band(s, threshold) for s in scores] + n_sc = bands.count("strong_clear") + n_su = bands.count("strong_uncertain") + n_wu = bands.count("weak_uncertain") + n_wc = bands.count("weak_clear") + n_cf_opus = sum(1 for t in turns if t["pick_cf"] == CAPABLE) + n_ef_opus = sum(1 for t in turns if t["pick_ef"] == CAPABLE) + rows_out.append({ + "run_id": run_id, + "task_name": task_name, + "reward": reward, + "n_turns": n, + "mean_score": round(mean(scores), 4), + "mean_confidence": round(mean(confs), 4), + "pct_strong_clear": round(n_sc / n, 4), + "pct_strong_uncertain": round(n_su / n, 4), + "pct_weak_uncertain": round(n_wu / n, 4), + "pct_weak_clear": round(n_wc / n, 4), + "opus_pct_cf": round(n_cf_opus / n, 4), + "nemotron_pct_cf": round(1 - n_cf_opus / n, 4), + "opus_pct_ef": round(n_ef_opus / n, 4), + "nemotron_pct_ef": round(1 - n_ef_opus / n, 4), + }) + + csv_path.parent.mkdir(parents=True, exist_ok=True) + with open(csv_path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames) + w.writeheader() + w.writerows(rows_out) + print(f"Per-task CSV → {csv_path} ({len(rows_out)} rows)") + + +async def score_run(run_dir: Path, recent_window: int, threshold: float) -> list[dict]: + jobs_dirs = list(run_dir.glob("jobs/*/")) or [run_dir] + run_id = run_dir.name + + all_rows: list[dict] = [] + for jobs_dir in jobs_dirs: + task_dirs = [d for d in jobs_dir.iterdir() if d.is_dir() and d.name != "verifier"] + print(f"{jobs_dir.name}: {len(task_dirs)} task dirs") + + for task_dir in sorted(task_dirs): + traj_path = task_dir / "agent" / "trajectory.json" + result_path = task_dir / "result.json" + if not traj_path.exists(): + continue + + traj = json.loads(traj_path.read_text()) + reward: float | None = None + task_name = task_dir.name + if result_path.exists(): + res = json.loads(result_path.read_text()) + task_name = res.get("task_name") or task_name + r = (res.get("verifier_result") or {}).get("rewards", {}).get("reward") + reward = float(r) if r is not None else None + + rows = await score_trajectory( + traj, reward, task_name, task_dir.name, run_id, recent_window, threshold + ) + all_rows.extend(rows) + print(f" {task_dir.name}: {len(rows)} turns, reward={reward}") + + return all_rows + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True, help="Path to run directory under benchmark/tb_runs/") + parser.add_argument("--output", help="Per-turn JSONL (default: /tmp/-scores.jsonl)") + parser.add_argument("--csv", help="Per-task CSV (default: /tmp/-per-task.csv)") + parser.add_argument("--threshold", type=float, default=0.20) + parser.add_argument("--window", type=int, default=RECENT_WINDOW) + args = parser.parse_args() + + run_dir = Path(args.run) + if not run_dir.exists(): + sys.exit(f"Run directory not found: {run_dir}") + + jsonl_out = Path(args.output or f"/tmp/{run_dir.name}-scores.jsonl") + csv_out = Path(args.csv or f"/tmp/{run_dir.name}-per-task.csv") + + all_rows = await score_run(run_dir, args.window, args.threshold) + + jsonl_out.parent.mkdir(parents=True, exist_ok=True) + with open(jsonl_out, "w") as fh: + for row in all_rows: + fh.write(json.dumps(row) + "\n") + print(f"Per-turn JSONL → {jsonl_out} ({len(all_rows)} rows)") + + if all_rows: + write_per_task_csv(all_rows, csv_out, args.threshold) + total = len(all_rows) + scores = [r["score"] for r in all_rows] + cf_opus = sum(1 for r in all_rows if r["pick_cf"] == CAPABLE) + ef_opus = sum(1 for r in all_rows if r["pick_ef"] == CAPABLE) + print(f"\nGlobal summary ({total} turns, threshold={args.threshold}):") + for bn in ["strong_clear", "strong_uncertain", "weak_uncertain", "weak_clear"]: + n = sum(1 for s in scores if band(s, args.threshold) == bn) + print(f" {bn:<22} {n:>5} ({100*n/total:.1f}%)") + print(f" cf → Opus: {cf_opus}/{total} ({100*cf_opus/total:.1f}%)") + print(f" ef → Opus: {ef_opus}/{total} ({100*ef_opus/total:.1f}%)") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 45c9fe04..32b75882 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -21,10 +21,19 @@ Those stages call for different amounts of model capability, which is what the router keys on. For each LLM call, stage-router estimates which stage the agent is in from the -**tool-result history** on the conversation: whether writes and edits are -landing, whether tests pass, whether commands error out, how much read-only -exploration is happening, and how far into the run it is. It turns those signals -into a confidence score for "this turn needs the capable tier", then routes: +**tool-result history** on the conversation, scoring two axes: + +- **WRONG → capable**: `severity` (windowed error severity), `spinning` (deep + churn with no reads or writes), and `exploring` (reading or planning without + producing) push toward the capable tier. +- **PROGRESS → efficient**: `recent_production_intensity` (writes and edits + landing over the recent window) pushes toward the efficient tier. + +The axes are **corroborative**: the signed score is `tanh`-squashed to a +confidence in `[0, 1]`, so one full signal alone scores ~`0.46` and a second +corroborating signal is what pushes it decisively past a `0.5` threshold. A +critical-error severity is a hard override that escalates on its own. The router +then routes: - the **capable** tier for uncertain, exploratory, or error-recovery turns, and - the **efficient** tier for settled, mechanical turns. @@ -89,14 +98,17 @@ clears the threshold to escalate to capable. (If you add the optional classifier, sub-threshold turns go to it instead of staying on the default tier.) -**Start with `0.5`.** It is the default and the recommended starting point. +**Set `0.5` explicitly.** It's the recommended starting point and what the +example below uses. When you omit the field the config default is `0.5` (for +both the profile config and the deprecated route bundle) — but setting it +explicitly keeps the intent clear. | `confidence_threshold` | Include `classifier:` block? | Typical use | |---|---|---| | `0.0` | no | Cost/latency-sensitive. Every signal-based verdict is accepted; no per-turn LLM call. Critical-error signals still escalate to capable. | -| `0.5` | no | Recommended starting point. A single strong signal clears the threshold on its own; derived from SWE-Bench Pro calibration. | +| `0.5` | no | Recommended starting point (config default). The scorer is corroborative — one full wrong signal scores ~`0.46`, just under `0.5` — so a decisive escalation takes a strong signal plus corroboration, while a critical error overrides regardless. Derived from SWE-Bench Pro Python-75 calibration. | | `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. | -| `1.0` | yes (required) | Classifier-driven. Equivalent to classifier-only `coding_agent` routing. | +| `1.0` | yes (required) | Classifier-driven. Equivalent to the legacy `coding_agent` profile. | The signal-vs-classifier split is dataset-dependent. Measure it in production via `routing_decisions.stage_router` on `/v1/stats` rather than relying on @@ -143,33 +155,29 @@ From the overlap tasks (those with both capable and efficient results): **Running the sweep** -Three scripts in `benchmark/calibration/stage_router/` form the pipeline: - -| Script | Input | Output | What it does | -|---|---|---|---| -| `signal_extractor.py` | Harbor task dir (JSONL trajectory) | one signal per turn | Replays a claude-code session, emitting the same signal the stage-router picker would have seen at each turn (write/edit/read counts, severity, tests passed, etc.) | -| `calibrate.py` | Harbor run dirs (one per arm) | `per_task.jsonl`, `per_turn.jsonl` | Reads `result.json` + trajectory JSONL for each task in each arm. Calls `signal_extractor` to build per-turn signals, then writes one record per task (outcome + features) and one record per turn (signal snapshot). Also prints RESCUE/LOSS/SAFE/HARD quadrant counts. | -| `sweep.py` | `per_task.jsonl`, `per_turn.jsonl` | Console table | Replays the per-turn signals through a set of escalation policies and scores each: pass%, escalation rate. The best-scoring policy that keeps escalation rate reasonable is your calibrated threshold. | +Replay your runs through the real Rust scorer and picker with +`benchmark/score_run.py` (the `switchyard-stage-router-scorer` skill). It emits +per-turn scores and per-task routing splits at a given threshold and window — +the actual `pick_capable_first` / `pick_efficient_first` decisions, not a +counterfactual: ```bash -cd benchmark/calibration/stage_router -python calibrate.py \ - --capable-run-dir /tmp/runs/your_capable_run \ - --efficient-run-dir /tmp/runs/your_efficient_probe -python sweep.py +# Score a probe run at a candidate threshold +uv run python benchmark/score_run.py --run benchmark/tb_runs/ \ + --threshold 0.5 --window 3 +# → /tmp/-scores.jsonl (per turn: score, confidence, pick_cf, pick_ef) +# → /tmp/-per-task.csv (per task: routing split, mean score/confidence) ``` -`calibrate.py` writes `per_task.jsonl` and `per_turn.jsonl`. `sweep.py` -prints the policy score table; pick the best row from that output. - -Pick the policy whose pass% beats `always_stay` with an acceptable -escalation rate. Translate it to a `confidence_threshold` value. A policy -that escalates ~20% of tasks maps roughly to `confidence_threshold: 0.5` -with `capable_first`. +Sweep a few candidate thresholds and read the routing split and pass rate off +the per-task CSV; the lowest threshold that rescues the RESCUE quadrant without +over-escalating the LOSS quadrant is your calibrated value. Because the scorer +is corroborative, a `0.5` threshold takes ~1.5 signals of agreement — a policy +that escalates ~20% of tasks maps roughly to `confidence_threshold: 0.5` with +`capable_first`. -Even 15–20 probe tasks produce a stable result because signal features are -extracted from the capable-arm trajectories, which are available for all -tasks from the pure-capable run. +Signals come from the actual picker replay, so even 15–20 probe tasks give a +stable result. **Caveat on efficient outcomes in stage-router vs. pure-efficient** @@ -209,6 +217,8 @@ switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` This is the recommended default: routing on tool signals alone, no classifier. +If you omit `confidence_threshold`, the config default of `0.5` applies; the +example sets it explicitly. `fallback_target_on_evict` is required and must reference one of the declared target ids. See [Context-Window Handling](../operations/context_window.md) for @@ -255,8 +265,9 @@ The route counts why each turn was routed the way it was under | Source | When | |---|---| -| `override` | A hard override fired (for example a critical error severity, or a clean test pass). | -| `dimensions` | The signals crossed `confidence_threshold` and picked the tier. | +| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | +| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — dropped the turn to the efficient tier. | +| `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | | `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | | `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured, so the default tier was used. | | `no_signal` | The request arrived before any tool-result history existed. | From 18052fbd0d447c25732c3ee20859eaa10093df1e Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 23 Jul 2026 16:22:44 -0700 Subject: [PATCH 15/21] refactor(stage_router): move scorer+picker into a shared Rust core; python calls it via bindings; drop dead prompt-classification path Signed-off-by: Sabhatina Selvam --- .../corpus_manifest.yaml | 58 -- .../extractors/__init__.py | 35 - .../extractors/harbor.py | 86 --- benchmark/dimension_calibration/run.py | 372 ----------- .../{score_run.py => score_staged_run.py} | 21 +- .../src/dimension_collector/config.rs | 351 ---------- .../src/dimension_collector/mod.rs | 129 +--- .../src/dimension_collector/scorers.rs | 602 ------------------ .../src/dimension_collector/tool_signals.rs | 246 +++---- crates/switchyard-components/src/lib.rs | 4 +- .../request_processors/dimension_collector.rs | 162 +---- .../switchyard-components/src/stage_router.rs | 353 ++++++++++ .../switchyard-py/src/component_bindings.rs | 2 + .../component_bindings/dimension_collector.rs | 213 +------ .../src/component_bindings/stage_router.rs | 157 +++++ .../lib/processors/stage_router/__init__.py | 20 +- .../lib/processors/stage_router/dimensions.py | 98 --- .../lib/processors/stage_router/picker.py | 106 +-- .../lib/processors/stage_router/scorer.py | 120 ---- switchyard_rust/components.py | 14 +- tests/test_dimension_collector.py | 177 ----- tests/test_stage_router_scorer.py | 143 ----- 22 files changed, 683 insertions(+), 2786 deletions(-) delete mode 100644 benchmark/dimension_calibration/corpus_manifest.yaml delete mode 100644 benchmark/dimension_calibration/extractors/__init__.py delete mode 100644 benchmark/dimension_calibration/extractors/harbor.py delete mode 100644 benchmark/dimension_calibration/run.py rename benchmark/{score_run.py => score_staged_run.py} (93%) delete mode 100644 crates/switchyard-components/src/dimension_collector/config.rs delete mode 100644 crates/switchyard-components/src/dimension_collector/scorers.rs create mode 100644 crates/switchyard-components/src/stage_router.rs create mode 100644 crates/switchyard-py/src/component_bindings/stage_router.rs delete mode 100644 switchyard/lib/processors/stage_router/dimensions.py delete mode 100644 switchyard/lib/processors/stage_router/scorer.py delete mode 100644 tests/test_dimension_collector.py delete mode 100644 tests/test_stage_router_scorer.py diff --git a/benchmark/dimension_calibration/corpus_manifest.yaml b/benchmark/dimension_calibration/corpus_manifest.yaml deleted file mode 100644 index a49ce7b5..00000000 --- a/benchmark/dimension_calibration/corpus_manifest.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Calibration corpus for the DimensionCollector. Each entry points at one -# Harbor run; the extractor turns that run into a normalized stream of -# ScoredPrompt records the calibration runner consumes. -# -# Paths are RELATIVE TO THE SWITCHYARD REPO ROOT. The runner resolves them -# against the repo so the same manifest works on any machine that has the -# referenced Harbor runs in the same place. -# -# Switchyard writes Harbor TBLite runs under benchmark/tb_runs/ (gitignored) -# via `switchyard verify --route-profile --model `. Each run -# lands at: -# -# benchmark/tb_runs//jobs//__/agent/ -# episode-0/prompt.txt -# episode-1/prompt.txt -# ... -# -# Point each entry below at the inner `jobs//` directory. -# -# To add your own run: -# 1. produce a Harbor run (`switchyard verify ...`) -# 2. add an entry below with a unique `id`, a meaningful `source` label, -# and the path to its `jobs//` directory -# 3. `uv run python benchmark/dimension_calibration/run.py` -# -# The per-dimension report under report/ is overwritten on every run. - -runs: - # Example entries — replace the paths with your local Harbor run directories. - # The runner skips entries whose path does not exist, so unused stubs are - # harmless. - - - id: stage_router-strong-default-deepseek - extractor: harbor - source: stage_router-strong-default-deepseek - path: "benchmark/tb_runs/tb-lite-stage_router-strong-default-deepseek/jobs/tb-lite-stage_router-strong-default-deepseek" - label: "StageRouter — capable_first picker, Opus + DeepSeek" - - - id: stage_router-strong-default-nemotron - extractor: harbor - source: stage_router-strong-default-nemotron - path: "benchmark/tb_runs/tb-lite-stage_router-strong-default-nemotron/jobs/tb-lite-stage_router-strong-default-nemotron" - label: "StageRouter — capable_first picker, Opus + Nemotron" - - - id: all-opus - extractor: harbor - source: all-opus - path: "benchmark/tb_runs/tb-lite-single-opus-4-7/jobs/tb-lite-single-opus-4-7" - label: "All-Opus baseline" - - - id: all-deepseek - extractor: harbor - source: all-deepseek - path: "benchmark/tb_runs/tb-lite-single-deepseek-v4-pro/jobs/tb-lite-single-deepseek-v4-pro" - label: "All-DeepSeek baseline" diff --git a/benchmark/dimension_calibration/extractors/__init__.py b/benchmark/dimension_calibration/extractors/__init__.py deleted file mode 100644 index dc0417c9..00000000 --- a/benchmark/dimension_calibration/extractors/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Trajectory-to-prompt extractors for the dimension-collector calibration corpus. - -Each extractor turns one harness run (currently: Harbor episode dirs) into a -stream of :class:`ScoredPrompt` records the calibration runner consumes. Two -scoring scopes per prompt: - -* ``full`` — the concatenated message blob the proxy actually sees in - production. System prompt, prior tool results, and tool calls all in. -* ``latest`` — the most recent user-role text content only. Cleaner signal - isolation; useful as a counterpoint to ``full``. -""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ScoredPrompt: - """Normalized prompt record consumed by the calibration runner. - - Fields are designed to be report-sliceable downstream: every analytical - cut in the report (per-source, per-run, per-turn-position) reads from - these columns and nothing else. - """ - - run_id: str - source: str # label from the manifest entry (e.g. "stage_router-settle-aware") - task: str # task id (e.g. "book-portfolio-analysis") - turn_idx: int # 0-based turn within the trajectory - full: str # full prompt blob (system + history + latest user) - latest: str # latest user-role text content only diff --git a/benchmark/dimension_calibration/extractors/harbor.py b/benchmark/dimension_calibration/extractors/harbor.py deleted file mode 100644 index 828d6e6a..00000000 --- a/benchmark/dimension_calibration/extractors/harbor.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Harbor episode dirs → ScoredPrompt stream. - -Harbor lays out one directory per task with an ``agent/`` subtree:: - - / - __/ - agent/ - episode-0/prompt.txt ← LLM-inbound prompt for turn 0 - episode-1/prompt.txt - ... - -Switchyard's ``switchyard verify --route-profile ...`` writes Harbor TBLite -runs at ``benchmark/tb_runs//jobs//`` — that's the -directory to point the manifest at. - -The ``source`` field on emitted records comes from the manifest entry's -``source:`` key (defaults to ``"harbor"``). Use distinct source labels on -different runs (e.g. ``stage_router-settle-aware``, ``all-opus``) so the by-source slice in -the report compares like-with-like. -""" - -from __future__ import annotations - -import re -from collections.abc import Iterator -from pathlib import Path - -from . import ScoredPrompt - -_EPISODE_DIR_PATTERN = re.compile(r"^episode-(\d+)$") -DEFAULT_SOURCE = "harbor" - - -def extract( - run_id: str, - run_root: Path, - *, - source: str = DEFAULT_SOURCE, -) -> Iterator[ScoredPrompt]: - """Walk a Harbor run directory and yield one ScoredPrompt per episode.""" - for task_dir in sorted(run_root.iterdir()): - if not task_dir.is_dir(): - continue - agent_dir = task_dir / "agent" - if not agent_dir.is_dir(): - continue - task = task_dir.name.split("__", 1)[0] - for episode_dir, idx in _ordered_episode_dirs(agent_dir): - prompt_path = episode_dir / "prompt.txt" - if not prompt_path.is_file(): - continue - try: - blob = prompt_path.read_text() - except OSError: - continue - if not blob.strip(): - continue - yield ScoredPrompt( - run_id=run_id, - source=source, - task=task, - turn_idx=idx, - full=blob, - # Harbor prompts are pre-concatenated blobs (system framing + - # tool history + latest turn), not a structured messages list. - # ``latest`` falls back to the same blob. - latest=blob, - ) - - -def _ordered_episode_dirs(agent_dir: Path) -> Iterator[tuple[Path, int]]: - """Yield (episode_dir, turn_idx) ordered by numeric episode index.""" - ordered: list[tuple[int, Path]] = [] - for child in agent_dir.iterdir(): - if not child.is_dir(): - continue - match = _EPISODE_DIR_PATTERN.match(child.name) - if not match: - continue - ordered.append((int(match.group(1)), child)) - ordered.sort(key=lambda pair: pair[0]) - for idx, path in ordered: - yield path, idx diff --git a/benchmark/dimension_calibration/run.py b/benchmark/dimension_calibration/run.py deleted file mode 100644 index 6bff437e..00000000 --- a/benchmark/dimension_calibration/run.py +++ /dev/null @@ -1,372 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tier 2 calibration runner. - -Walks ``corpus_manifest.yaml``, calls the per-extractor module for each run, -and scores every prompt twice (``full`` and ``latest`` scope) through the -PyO3-backed :class:`switchyard_rust.components.DimensionCollector`. - -Emits under ``report/``: - -* ``data.csv`` — every prompt × scope × dimension score, one row. -* ``tier2_overall.md`` — per-dimension headline numbers. -* ``tier2_by_source.md`` — per-dimension fire rate, sliced by manifest source. -* ``tier2_by_turn.md`` — bucketed turn-index growth slices. - -Run with:: - - uv run python benchmark/dimension_calibration/run.py - -Paths in the manifest resolve relative to the repo root, so adding a new -Harbor run to the corpus is one YAML entry pointing at the run directory. -""" - -from __future__ import annotations - -import asyncio -import csv -import importlib -import statistics -import sys -import time -from collections import Counter, defaultdict -from collections.abc import Iterable, Iterator -from dataclasses import asdict -from pathlib import Path -from typing import Any - -import yaml -from extractors import ScoredPrompt - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.components import ( - DimensionCollector, - get_context_signals, -) -from switchyard_rust.core import ChatRequest - -HERE = Path(__file__).resolve().parent -REPO_ROOT = HERE.parents[1] -MANIFEST_PATH = HERE / "corpus_manifest.yaml" -REPORT_DIR = HERE / "report" - - -def load_manifest() -> list[dict[str, Any]]: - """Load the manifest and resolve paths relative to the repo root.""" - raw = yaml.safe_load(MANIFEST_PATH.read_text()) - runs: list[dict[str, Any]] = [] - for entry in raw.get("runs", []): - run = dict(entry) - run["path"] = str((REPO_ROOT / entry["path"]).resolve()) - runs.append(run) - return runs - - -def stream_prompts(runs: list[dict[str, Any]]) -> Iterator[ScoredPrompt]: - """Walk all runs in the manifest and yield ScoredPrompts in order.""" - for run in runs: - module = importlib.import_module(f"extractors.{run['extractor']}") - path = Path(run["path"]) - if not path.exists(): - print( - f" skip {run['id']}: path does not exist ({path})", - file=sys.stderr, - ) - continue - kwargs: dict[str, Any] = {} - if "source" in run: - kwargs["source"] = run["source"] - yield from module.extract(run["id"], path, **kwargs) - - -def make_request(text: str) -> ChatRequest: - """Wrap a plain text blob as the user message of an OpenAI-chat request. - - The DimensionCollector adapter extracts the user content from - ``messages[]``, so we shape the request body to match what production - traffic looks like at the proxy boundary. - """ - return ChatRequest.openai_chat({ - "model": "calibration", - "messages": [{"role": "user", "content": text}], - }) - - -async def score_one(collector: DimensionCollector, text: str) -> dict[str, Any]: - """Score a single prompt and return the signals as a flat dict.""" - ctx = ProxyContext() - await collector.process(ctx, make_request(text)) - signals = get_context_signals(ctx) - if signals is None: - return {"token_count_estimate": 0, "dims": {}} - return { - "token_count_estimate": signals.token_count_estimate, - "dims": {dim.name: dim.score for dim in signals.dimensions}, - } - - -async def score_all(prompts: Iterable[ScoredPrompt]) -> list[dict[str, Any]]: - """Score every prompt twice (full + latest scopes) and return flat rows.""" - # No-arg constructor picks up the Rust-side `ScoringConfig::default()` — - # the populated keyword set. Passing `ScoringConfig()` from Python would - # build an explicit-empty config and bypass the Rust default. - collector = DimensionCollector() - rows: list[dict[str, Any]] = [] - count = 0 - started_at = time.perf_counter() - for prompt in prompts: - for scope in ("full", "latest"): - text = prompt.full if scope == "full" else prompt.latest - result = await score_one(collector, text) - row = { - **asdict(prompt), - "scope": scope, - "char_len": len(text), - "token_count_estimate": result["token_count_estimate"], - **{f"dim:{name}": score for name, score in result["dims"].items()}, - } - # Trim raw text out of the CSV — keep only metadata. The raw - # prompts live in the source trajectories; replicating them - # here would balloon the CSV and serve no analytical purpose. - row.pop("full", None) - row.pop("latest", None) - rows.append(row) - count += 1 - if count % 500 == 0: - elapsed = time.perf_counter() - started_at - print( - f" scored {count} prompts ({elapsed:.1f}s, " - f"{count / elapsed:.0f}/s)", - file=sys.stderr, - ) - elapsed = time.perf_counter() - started_at - print( - f" scored {count} prompts total in {elapsed:.1f}s " - f"({count / max(elapsed, 1e-9):.0f}/s)", - file=sys.stderr, - ) - return rows - - -# ─── Report writers ────────────────────────────────────────────────────────── - -# Canonical dimension order as the collector emits them, mirrored here so the -# report tables read consistently across runs. -DIMENSIONS = [ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", -] - - -def write_csv(rows: list[dict[str, Any]]) -> Path: - """Dump every (prompt, scope, dimension) row to CSV for offline analysis.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "data.csv" - fieldnames = [ - "run_id", "source", "task", "turn_idx", "scope", "char_len", - "token_count_estimate", - *[f"dim:{d}" for d in DIMENSIONS], - ] - with out.open("w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - writer.writerows(rows) - return out - - -def fire_stats(rows: list[dict[str, Any]], dim: str) -> dict[str, Any]: - """Compute fire-rate + nonzero-score summary for one dimension.""" - key = f"dim:{dim}" - fires: list[float] = [] - total = 0 - for row in rows: - total += 1 - score = row.get(key, 0.0) or 0.0 - if score != 0.0: - fires.append(score) - if total == 0: - return {"n": 0, "fires": 0, "rate": 0.0, - "nonzero_min": None, "nonzero_p50": None, "nonzero_max": None} - return { - "n": total, - "fires": len(fires), - "rate": len(fires) / total, - "nonzero_min": min(fires) if fires else None, - "nonzero_p50": statistics.median(fires) if fires else None, - "nonzero_max": max(fires) if fires else None, - } - - -def write_overall(rows: list[dict[str, Any]]) -> Path: - """Per-dimension headline: fire-rate + nonzero score band, by scope.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_overall.md" - lines: list[str] = [] - lines.append("# Tier-2 calibration — overall per-dimension fire rates") - lines.append("") - lines.append(f"Total prompt observations: **{len(rows)}** " - f"(prompts × 2 scopes).") - lines.append("") - for scope in ("full", "latest"): - scoped = [r for r in rows if r["scope"] == scope] - lines.append(f"## scope = `{scope}` ({len(scoped)} rows)") - lines.append("") - lines.append("| Dimension | Fire rate | Fires | Nonzero min / p50 / max |") - lines.append("|---|---:|---:|---|") - for dim in DIMENSIONS: - s = fire_stats(scoped, dim) - band = "—" if s["fires"] == 0 else \ - f"{s['nonzero_min']:.2f} / {s['nonzero_p50']:.2f} / {s['nonzero_max']:.2f}" - lines.append( - f"| `{dim}` | {s['rate']:.1%} | {s['fires']} | {band} |" - ) - lines.append("") - # Aggregate scalars on the same scope - tc = [r["token_count_estimate"] for r in scoped] - if tc: - lines.append( - f"`token_count_estimate`: min={min(tc)}, " - f"p50={statistics.median(tc):.0f}, max={max(tc)}, " - f"mean={statistics.fmean(tc):.0f}" - ) - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_by_source(rows: list[dict[str, Any]]) -> Path: - """Per-dimension fire-rate sliced by trajectory source (hermes vs harbor).""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_by_source.md" - sources = sorted({r["source"] for r in rows}) - lines: list[str] = [] - lines.append("# Tier-2 calibration — fire rate by source") - lines.append("") - lines.append("Restricted to `scope=full` (production-realistic blob).") - lines.append("") - header = "| Dimension | " + " | ".join(sources) + " |" - sep = "|---|" + "|".join(["---:"] * len(sources)) + "|" - lines.append(header) - lines.append(sep) - for dim in DIMENSIONS: - per_source: list[str] = [] - for src in sources: - scoped = [r for r in rows if r["source"] == src and r["scope"] == "full"] - s = fire_stats(scoped, dim) - per_source.append(f"{s['rate']:.1%}") - lines.append(f"| `{dim}` | " + " | ".join(per_source) + " |") - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_by_turn(rows: list[dict[str, Any]]) -> Path: - """Fire-rate sliced by turn-position bucket — diagnoses trajectory growth.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_by_turn.md" - # Three buckets: turn 0 (task framing), early (1-4), mid+ (5+). - def bucket(idx: int) -> str: - if idx == 0: - return "turn=0" - if idx <= 4: - return "1-4" - return "5+" - - lines: list[str] = [] - lines.append("# Tier-2 calibration — fire rate by turn position") - lines.append("") - lines.append("Restricted to `scope=full`. Buckets: `turn=0` (task framing) / " - "`1-4` (early loop) / `5+` (deep loop).") - lines.append("") - buckets = ["turn=0", "1-4", "5+"] - header = "| Dimension | " + " | ".join(buckets) + " |" - lines.append(header) - lines.append("|---|" + "|".join(["---:"] * len(buckets)) + "|") - for dim in DIMENSIONS: - per_bucket: list[str] = [] - for b in buckets: - scoped = [ - r for r in rows - if r["scope"] == "full" and bucket(r["turn_idx"]) == b - ] - s = fire_stats(scoped, dim) - per_bucket.append(f"{s['rate']:.1%}") - lines.append(f"| `{dim}` | " + " | ".join(per_bucket) + " |") - lines.append("") - # Per-bucket row counts for context - counts: Counter[str] = Counter( - bucket(r["turn_idx"]) for r in rows if r["scope"] == "full" - ) - lines.append( - "Row counts per bucket (scope=full): " - + ", ".join(f"{b}={counts.get(b, 0)}" for b in buckets) - ) - lines.append("") - out.write_text("\n".join(lines)) - return out - - -def write_per_run_breakdown(rows: list[dict[str, Any]]) -> Path: - """Per-run row counts so reviewers see corpus composition.""" - REPORT_DIR.mkdir(parents=True, exist_ok=True) - out = REPORT_DIR / "tier2_corpus_composition.md" - by_run: defaultdict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - if row["scope"] == "full": - by_run[row["run_id"]].append(row) - lines: list[str] = [] - lines.append("# Tier-2 calibration — corpus composition") - lines.append("") - lines.append("| Run | Prompts (scope=full) | Tasks | Avg turn idx |") - lines.append("|---|---:|---:|---:|") - for run_id in sorted(by_run): - scoped = by_run[run_id] - tasks = len({r["task"] for r in scoped}) - avg_turn = statistics.fmean(r["turn_idx"] for r in scoped) - lines.append(f"| `{run_id}` | {len(scoped)} | {tasks} | {avg_turn:.1f} |") - lines.append("") - out.write_text("\n".join(lines)) - return out - - -async def main() -> int: - print("loading manifest…", file=sys.stderr) - runs = load_manifest() - print(f" {len(runs)} runs declared", file=sys.stderr) - - print("scoring prompts (this is one DimensionCollector pass per scope)…", - file=sys.stderr) - rows = await score_all(stream_prompts(runs)) - - print("writing reports…", file=sys.stderr) - csv_path = write_csv(rows) - overall_path = write_overall(rows) - by_source_path = write_by_source(rows) - by_turn_path = write_by_turn(rows) - composition_path = write_per_run_breakdown(rows) - - print("done.", file=sys.stderr) - print() - print(f" csv: {csv_path.relative_to(REPO_ROOT)}") - print(f" overall report: {overall_path.relative_to(REPO_ROOT)}") - print(f" by-source report: {by_source_path.relative_to(REPO_ROOT)}") - print(f" by-turn report: {by_turn_path.relative_to(REPO_ROOT)}") - print(f" corpus composition: {composition_path.relative_to(REPO_ROOT)}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(asyncio.run(main())) diff --git a/benchmark/score_run.py b/benchmark/score_staged_run.py similarity index 93% rename from benchmark/score_run.py rename to benchmark/score_staged_run.py index f34e4189..988e2d59 100644 --- a/benchmark/score_run.py +++ b/benchmark/score_staged_run.py @@ -20,16 +20,16 @@ from pathlib import Path from statistics import mean -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from switchyard.lib.processors.stage_router.dimensions import from_signal from switchyard.lib.processors.stage_router.picker import ( CAPABLE, pick_capable_first, pick_efficient_first, ) -from switchyard.lib.processors.stage_router.scorer import score as scorer_score -from switchyard_rust.components import DimensionCollector, get_tool_result_signal +from switchyard_rust.components import ( + DimensionCollector, + get_tool_result_signal, + stage_score_signal, +) from switchyard_rust.core import ChatRequest, ProxyContext RECENT_WINDOW = 3 @@ -112,11 +112,10 @@ async def score_trajectory( if signal is None: continue - # Raw score for analysis: wrong signals score positive (→CAPABLE), - # progress signals negative (→EFFICIENT). Picker-independent, so the + # Raw score for analysis (Rust scorer): wrong signals score positive + # (→CAPABLE), progress negative (→EFFICIENT). Picker-independent, so the # histogram shows the capable/efficient separation directly. - dims = from_signal(signal) - sr = scorer_score(dims) # fixed weights; threshold dials corroboration + score, confidence = stage_score_signal(signal) # Actual picker decisions on the same ctx — both just read the signal, # no extra dc.process() calls needed @@ -129,8 +128,8 @@ async def score_trajectory( "run_id": run_id, "reward": reward, "turn_depth": signal.turn_depth, - "score": sr.score, - "confidence": sr.confidence, + "score": score, + "confidence": confidence, "tool_name": tool_name, "is_error": is_error, "write_count": signal.write_count, diff --git a/crates/switchyard-components/src/dimension_collector/config.rs b/crates/switchyard-components/src/dimension_collector/config.rs deleted file mode 100644 index d0c93cdc..00000000 --- a/crates/switchyard-components/src/dimension_collector/config.rs +++ /dev/null @@ -1,351 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Config types and scored-signal records for the dimension collector. -//! -//! Mirrors `ClawRouter/src/router/types.ts` (MIT). Only the fields the -//! collector itself consumes are reproduced here; tier maps, promotions, -//! and model pricing live with the eventual rules estimator, not the -//! context-signal layer. - -/// One scorer's output: dimension name, signed score, optional human-readable signal. -/// -/// Scores roughly in `[-1, 1]`. Individual scorers narrow that range -/// (`simple_indicators` is non-positive, `code_presence` is non-negative). -#[derive(Clone, Debug, PartialEq)] -pub struct DimensionScore { - pub name: &'static str, - pub score: f32, - pub signal: Option, -} - -impl DimensionScore { - /// Convenience constructor for the no-signal case (score 0.0). - pub fn zero(name: &'static str) -> Self { - Self { - name, - score: 0.0, - signal: None, - } - } -} - -/// Token-count thresholds for `score_token_count`. -/// -/// Below `short` → score `-1.0` (looks SIMPLE). Above `long` → score -/// `+1.0` (looks COMPLEX). In-between → `0.0`. -#[derive(Clone, Debug, PartialEq)] -pub struct TokenCountThresholds { - pub short: u32, - pub long: u32, -} - -impl Default for TokenCountThresholds { - fn default() -> Self { - Self { - short: 50, - long: 500, - } - } -} - -/// Pre-lowercased keyword list for substring-match scorers. -/// -/// Keyword lists are lowercased once at config construction so the -/// per-request hot path does only the prompt-side `to_lowercase()` plus -/// `O(len(keywords) × len(text))` substring scans. -#[derive(Clone, Debug, Default, PartialEq)] -pub struct Keywords(Vec); - -impl Keywords { - /// Constructs a lowercased keyword set from any iterator of strings. - pub fn new(items: impl IntoIterator>) -> Self { - let lowered: Vec = items - .into_iter() - .map(|item| item.as_ref().to_lowercase()) - .collect(); - Self(lowered) - } - - /// Constructs a keyword set from a static list of already-lowercase - /// `&'static str` entries — used by [`ScoringConfig::clawrouter_defaults`] - /// so the per-construction lowercase pass is skipped. - pub fn from_static(items: &[&'static str]) -> Self { - Self(items.iter().map(|s| (*s).to_string()).collect()) - } - - /// Returns the underlying lowercased keyword slice. - pub fn as_slice(&self) -> &[String] { - &self.0 - } - - /// Returns whether the keyword set is empty. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -/// Per-scorer configuration consumed by the dimension collector. -/// -/// Only the subset of `ScoringConfig` the collector needs lives here. -/// Tier boundaries, dimension weights, confidence calibration, and -/// model-pricing knobs belong to a separate rules estimator — putting -/// them here would couple the context-signal layer to scoring policy. -#[derive(Clone, Debug, PartialEq)] -pub struct ScoringConfig { - pub token_count: TokenCountThresholds, - pub code_keywords: Keywords, - pub reasoning_keywords: Keywords, - pub simple_keywords: Keywords, - pub technical_keywords: Keywords, - pub creative_keywords: Keywords, - pub imperative_verbs: Keywords, - pub constraint_indicators: Keywords, - pub output_format_keywords: Keywords, - pub reference_keywords: Keywords, - pub negation_keywords: Keywords, - pub domain_specific_keywords: Keywords, -} - -impl Default for ScoringConfig { - fn default() -> Self { - Self::clawrouter_defaults() - } -} - -impl ScoringConfig { - /// Returns the ClawRouter-aligned default keyword set. - /// - /// Trailing-space suffixes (e.g. `"def "`, `"not "`) prevent false - /// substring matches inside common longer words (`"definition"`, - /// `"another"`). The collector lower-cases the prompt once per request - /// and matches each keyword as a substring, so suffix discipline is - /// the cheapest way to keep precision up. - pub fn clawrouter_defaults() -> Self { - Self { - token_count: TokenCountThresholds::default(), - code_keywords: Keywords::from_static(defaults::CODE), - reasoning_keywords: Keywords::from_static(defaults::REASONING), - simple_keywords: Keywords::from_static(defaults::SIMPLE), - technical_keywords: Keywords::from_static(defaults::TECHNICAL), - creative_keywords: Keywords::from_static(defaults::CREATIVE), - imperative_verbs: Keywords::from_static(defaults::IMPERATIVE_VERBS), - constraint_indicators: Keywords::from_static(defaults::CONSTRAINT_INDICATORS), - output_format_keywords: Keywords::from_static(defaults::OUTPUT_FORMAT), - reference_keywords: Keywords::from_static(defaults::REFERENCE), - negation_keywords: Keywords::from_static(defaults::NEGATION), - domain_specific_keywords: Keywords::from_static(defaults::DOMAIN_SPECIFIC), - } - } -} - -/// Static keyword tables for [`ScoringConfig::clawrouter_defaults`]. -/// All entries lowercase by construction; retuning here is a pure data diff. -mod defaults { - pub static CODE: &[&str] = &[ - "def ", - "class ", - "function", - "async ", - "await ", - "import ", - "from ", - "return ", - "fn ", - "struct ", - "impl ", - "trait ", - "pub ", - "mod ", - "package ", - "interface ", - "namespace ", - "let ", - "const ", - "```", - "->", - "=>", - "{}", - "[]", - ]; - - pub static REASONING: &[&str] = &[ - "prove", - "theorem", - "derive", - "step by step", - "let's think", - "reasoning", - "deduce", - "induction", - "lemma", - "proof", - "show that", - "demonstrate", - "given that", - "therefore", - "hence ", - "thus ", - "implies", - "because of", - ]; - - // "no " and "ok " removed — collide with "no longer", "no need", "ok with". - pub static SIMPLE: &[&str] = &[ - "hello", - "hi ", - "thanks", - "thank you", - "what is", - "what's", - "define ", - "meaning of", - "summary of", - "translate ", - ]; - - pub static TECHNICAL: &[&str] = &[ - "distributed", - "concurrent", - "kubernetes", - "container", - "algorithm", - "protocol", - "throughput", - "latency", - "bandwidth", - "consistency", - "compiler", - "interpreter", - "garbage collection", - "memory leak", - "race condition", - "deadlock", - "cache", - "transaction", - "atomic", - "encryption", - "compression", - "deserialize", - "serialize", - ]; - - pub static CREATIVE: &[&str] = &[ - "story", - "poem", - "novel", - "lyrics", - "creative", - "brainstorm", - "imagine", - "fictional", - "narrative", - "character", - "plot", - "metaphor", - "analogy", - "essay", - ]; - - // Over-common verbs ("add", "update", "change", "make", "write") - // removed — they appear in nearly every request and gave the dimension - // no discriminating power. - pub static IMPERATIVE_VERBS: &[&str] = &[ - "implement ", - "design ", - "refactor ", - "architect ", - "engineer ", - "compose ", - "scaffold ", - "bootstrap ", - ]; - - pub static CONSTRAINT_INDICATORS: &[&str] = &[ - "must ", - "should ", - "at most", - "at least", - "within", - "no more than", - "fewer than", - "exactly", - "o(n)", - "o(1)", - "o(log", - "linear time", - "constant time", - "bounded by", - "must not", - "should not", - "ensure ", - ]; - - pub static OUTPUT_FORMAT: &[&str] = &[ - "json", - "yaml", - "csv", - "xml", - "html", - "markdown", - "table", - "format as", - "in the format", - "output as", - "structure", - "schema", - ]; - - pub static REFERENCE: &[&str] = &[ - "the code above", - "the code below", - "the file", - "the function", - "the api docs", - "the previous", - "earlier you said", - "in the last", - "as shown above", - "above mentioned", - "as discussed", - "you mentioned", - ]; - - pub static NEGATION: &[&str] = &[ - "not ", - "never ", - "neither ", - "except ", - "unless ", - "without ", - "no longer", - "cannot ", - "couldn't ", - "shouldn't ", - "wouldn't ", - "doesn't ", - "don't ", - "didn't ", - ]; - - pub static DOMAIN_SPECIFIC: &[&str] = &[ - "quantum", - "fpga", - "embedded", - "kernel", - "driver", - "genomics", - "blockchain", - "smart contract", - "calculus", - "differential", - "tensor", - "linear algebra", - "signal processing", - "fourier", - "cryptography", - "graph theory", - "topology", - "category theory", - ]; -} diff --git a/crates/switchyard-components/src/dimension_collector/mod.rs b/crates/switchyard-components/src/dimension_collector/mod.rs index ae42f22e..b33b31bb 100644 --- a/crates/switchyard-components/src/dimension_collector/mod.rs +++ b/crates/switchyard-components/src/dimension_collector/mod.rs @@ -1,135 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Context-signal extraction layer. +//! Signal extraction for the stage_router. //! -//! This module owns the pure logic for turning a request's text content -//! into a tuple of scored dimensions plus a token-count estimate and an -//! agentic scalar. The [`crate::request_processors::dimension_collector`] -//! module wraps it as a request-side Switchyard component. -//! -//! Port of `switchyard.experimental.rules_routing.dimension_collector` -//! (deleted in e5f88be2). This layer fits into a broader context-signal -//! vs estimated-signal taxonomy. +//! Owns the pure logic for reading a coding-agent request's tool-call history +//! into a [`ToolResultSignal`] (write/edit/read counts, error severity, streaks), +//! plus response-side signals. The +//! [`crate::request_processors::dimension_collector`] module wraps it as a +//! request-side Switchyard component. -pub mod config; pub mod response; -pub mod scorers; pub mod tool_signals; -pub use config::{DimensionScore, Keywords, ScoringConfig, TokenCountThresholds}; pub use response::{extract_response_signals, ResponseFlag, ResponseSignals}; pub use tool_signals::{ extract_tool_signals, extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, }; - -/// Token-per-character heuristic from ClawRouter (`strategy.ts`): -/// `estimatedTokens = ceil(fullText.length / 4)`. Cheap and good enough -/// for routing decisions. A real tokenizer lands behind a feature flag -/// in future work. -pub const CHARS_PER_TOKEN: u32 = 4; - -/// Estimate token count using the ClawRouter chars/4 heuristic. -pub fn estimate_token_count(text: &str) -> u32 { - let chars = text.chars().count() as u32; - chars.div_ceil(CHARS_PER_TOKEN) -} - -/// Aggregate output of one dimension-collection pass. -/// -/// Stamped into `ProxyContext` by the request-processor adapter so -/// downstream estimators (LLM classifier, rules estimator, …) can read -/// dimension scores without re-extracting them. -/// -/// Phase D removed the previously-emitted `agentic_score` scalar after -/// Calibration showed the underlying signal could not be -/// stably extracted from real TBLite traffic. Downstream consumers that -/// previously read `agentic_score` should use a context-level signal -/// (tool-call count, scope-aware routing input) instead. -#[derive(Clone, Debug, PartialEq)] -pub struct ContextSignals { - pub dimensions: Vec, - pub token_count_estimate: u32, -} - -/// Run all 14 dimension scorers against a prompt. -/// -/// `lower_text` MUST be the lowercased version of the prompt. The raw -/// `prompt` is also accepted so `score_question_complexity` can count -/// `?` against the case-insensitive original (case doesn't matter for -/// `?`, but the unfolded text avoids re-walking the lower_text). -/// -/// Emits a `ContextSignals` carrying: -/// -/// * the 14 dimension scores in the canonical order documented below, -/// * a `chars/4` token-count estimate. -pub fn extract_signals(prompt: &str, lower_text: &str, config: &ScoringConfig) -> ContextSignals { - let token_count_estimate = estimate_token_count(lower_text); - - let dimensions = vec![ - scorers::score_token_count( - token_count_estimate, - config.token_count.short, - config.token_count.long, - ), - scorers::score_code_presence(lower_text, config.code_keywords.as_slice()), - scorers::score_reasoning_markers(lower_text, config.reasoning_keywords.as_slice()), - scorers::score_technical_terms(lower_text, config.technical_keywords.as_slice()), - scorers::score_creative_markers(lower_text, config.creative_keywords.as_slice()), - scorers::score_simple_indicators(lower_text, config.simple_keywords.as_slice()), - scorers::score_imperative_verbs(lower_text, config.imperative_verbs.as_slice()), - scorers::score_constraint_count(lower_text, config.constraint_indicators.as_slice()), - scorers::score_output_format(lower_text, config.output_format_keywords.as_slice()), - scorers::score_reference_complexity(lower_text, config.reference_keywords.as_slice()), - scorers::score_negation_complexity(lower_text, config.negation_keywords.as_slice()), - scorers::score_domain_specificity(lower_text, config.domain_specific_keywords.as_slice()), - scorers::score_multi_step_patterns(lower_text), - scorers::score_question_complexity(prompt), - ]; - - ContextSignals { - dimensions, - token_count_estimate, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn estimate_token_count_uses_chars_div_four_ceiling() { - assert_eq!(estimate_token_count(""), 0); - assert_eq!(estimate_token_count("a"), 1); - assert_eq!(estimate_token_count("abcd"), 1); - assert_eq!(estimate_token_count("abcde"), 2); - } - - #[test] - fn extract_signals_runs_all_fourteen_scorers_in_canonical_order() { - let config = ScoringConfig::default(); - let prompt = "Hello world."; - let signals = extract_signals(prompt, &prompt.to_lowercase(), &config); - - let names: Vec<&str> = signals.dimensions.iter().map(|dim| dim.name).collect(); - assert_eq!( - names, - vec![ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", - ], - ); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/scorers.rs b/crates/switchyard-components/src/dimension_collector/scorers.rs deleted file mode 100644 index 454360ed..00000000 --- a/crates/switchyard-components/src/dimension_collector/scorers.rs +++ /dev/null @@ -1,602 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Pure dimension scorers — `(text, keywords, knobs) -> DimensionScore`. -//! -//! Direct port of the 15 scorer functions in -//! `switchyard.experimental.rules_routing.scorers` (deleted in e5f88be2), -//! itself a port of `ClawRouter/src/router/rules.ts` (MIT). -//! -//! All functions are pure: no shared state, no I/O. The collector -//! lowercases the input text once per request and lowercases the -//! keyword sets once at construction (see [`crate::dimension_collector::Keywords`]), -//! so the per-request hot path is just integer arithmetic and substring scans. - -use super::config::DimensionScore; - -/// Score the `tokenCount` dimension. -/// -/// Below `short` → `-1.0` (looks SIMPLE). Above `long` → `+1.0` -/// (looks COMPLEX). Otherwise `0.0`. Mirrors `scoreTokenCount` in -/// `rules.ts` verbatim. -pub fn score_token_count(estimated_tokens: u32, short: u32, long: u32) -> DimensionScore { - if estimated_tokens < short { - return DimensionScore { - name: "tokenCount", - score: -1.0, - signal: Some(format!("short ({estimated_tokens} tokens)")), - }; - } - if estimated_tokens > long { - return DimensionScore { - name: "tokenCount", - score: 1.0, - signal: Some(format!("long ({estimated_tokens} tokens)")), - }; - } - DimensionScore::zero("tokenCount") -} - -/// Generic keyword-match scorer used by most dimensions. -/// -/// Counts how many `lower_keywords` appear as substrings in `lower_text`. -/// Returns `score_high` once the count reaches `high`, `score_low` once -/// it reaches `low`, otherwise `0.0`. Mirrors `scoreKeywordMatch` in -/// `rules.ts`. -/// -/// Both inputs MUST already be lower-cased. -// -// The 8-arg signature is intentional: this is the shared kernel for 10 -// of the 15 dimension scorers. Each caller is a 1-line specialization -// that pins the per-dimension knobs (name, label, thresholds, scores). -// Bundling into a config struct would force every call site through a -// constructor + clone with no readability win. -#[allow(clippy::too_many_arguments)] -pub fn score_keyword_match( - lower_text: &str, - lower_keywords: &[String], - name: &'static str, - signal_label: &str, - low: usize, - high: usize, - score_low: f32, - score_high: f32, -) -> DimensionScore { - let matches: Vec<&str> = lower_keywords - .iter() - .filter_map(|kw| lower_text.contains(kw.as_str()).then_some(kw.as_str())) - .collect(); - - let count = matches.len(); - if count >= high { - return DimensionScore { - name, - score: score_high, - signal: Some(format!("{signal_label} ({})", preview_matches(&matches),)), - }; - } - if count >= low { - return DimensionScore { - name, - score: score_low, - signal: Some(format!("{signal_label} ({})", preview_matches(&matches),)), - }; - } - DimensionScore::zero(name) -} - -/// Score `codePresence`: how code-shaped does the prompt look? -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 1.0)`. -pub fn score_code_presence(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "codePresence", - "code", - 1, - 2, - 0.5, - 1.0, - ) -} - -/// Score `reasoningMarkers`: how proof / chain-of-thought shaped? -/// -/// Thresholds `(low=1, high=2)`, scores `(0.7, 1.0)`. `score_low` of -/// 0.7 is deliberately high — one reasoning marker is a strong signal. -pub fn score_reasoning_markers(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "reasoningMarkers", - "reasoning", - 1, - 2, - 0.7, - 1.0, - ) -} - -/// Score `technicalTerms` (kubernetes, distributed, algorithm, …). -/// -/// Thresholds `(low=2, high=4)`, scores `(0.5, 1.0)` — needs more -/// matches than other keyword dimensions because technical jargon is -/// noisier and a single hit means less. -pub fn score_technical_terms(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "technicalTerms", - "technical", - 2, - 4, - 0.5, - 1.0, - ) -} - -/// Score `creativeMarkers` (story, poem, brainstorm, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 0.7)` — capped below -/// code / reasoning ceilings because creative requests aren't always -/// complex. -pub fn score_creative_markers(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "creativeMarkers", - "creative", - 1, - 2, - 0.5, - 0.7, - ) -} - -/// Score `simpleIndicators` (what is, hello, define, …). -/// -/// ALWAYS negative: matches push the request toward SIMPLE rather -/// than away from any other tier. Scores `(-1.0, -1.0)`. -pub fn score_simple_indicators(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "simpleIndicators", - "simple", - 1, - 2, - -1.0, - -1.0, - ) -} - -/// Score `imperativeVerbs` (implement, design, refactor, …). -/// -/// Thresholds `(low=2, high=3)`, scores `(0.3, 0.5)` — mild signal; -/// imperative verbs are common across all tiers so weights are low. -/// **Phase B retune**: thresholds raised from `(1, 2)` to `(2, 3)` after -/// Calibration showed a 97% baseline fire rate. Multiple -/// strong verbs now required before the signal fires. -pub fn score_imperative_verbs(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "imperativeVerbs", - "imperative", - 2, - 3, - 0.3, - 0.5, - ) -} - -/// Score `constraintCount` (at most, within, O(), …). -/// -/// Thresholds `(low=1, high=3)`, scores `(0.3, 0.7)` — high score -/// requires multiple constraints; prompts with many constraints tend -/// to be COMPLEX/REASONING-tier algorithmic work. -pub fn score_constraint_count(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "constraintCount", - "constraints", - 1, - 3, - 0.3, - 0.7, - ) -} - -/// Score `outputFormat` (json, yaml, table, csv, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.4, 0.7)` — structured -/// output correlates with structured tasks (parsing, transformation) -/// which skew MEDIUM/COMPLEX. -pub fn score_output_format(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "outputFormat", - "format", - 1, - 2, - 0.4, - 0.7, - ) -} - -/// Score `referenceComplexity` ("the code above", "the API docs", …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.3, 0.5)` — mild signal -/// that the request depends on multi-turn context. -pub fn score_reference_complexity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "referenceComplexity", - "references", - 1, - 2, - 0.3, - 0.5, - ) -} - -/// Score `negationComplexity` (not, never, except, unless, …). -/// -/// Thresholds `(low=2, high=3)`, scores `(0.3, 0.5)` — needs multiple -/// negations to fire; many real prompts contain at least one negation -/// so a single hit doesn't move the needle. -pub fn score_negation_complexity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "negationComplexity", - "negation", - 2, - 3, - 0.3, - 0.5, - ) -} - -/// Score `domainSpecificity` (quantum, FPGA, genomics, …). -/// -/// Thresholds `(low=1, high=2)`, scores `(0.5, 0.8)` — even one -/// domain-specific term is a strong signal that the request needs a -/// capable model. -pub fn score_domain_specificity(lower_text: &str, lower_keywords: &[String]) -> DimensionScore { - score_keyword_match( - lower_text, - lower_keywords, - "domainSpecificity", - "domain-specific", - 1, - 2, - 0.5, - 0.8, - ) -} - -/// Score `multiStepPatterns` (`first…then`, `step N`, numbered lists). -/// -/// Three byte-level patterns from `scoreMultiStep` in `rules.ts`, -/// inlined as manual scans to avoid the regex dependency and the -/// runtime fallibility of `Regex::new`. Any hit → score `0.5`; no -/// hit → `0.0`. Caller passes already-lower-cased text. -pub fn score_multi_step_patterns(lower_text: &str) -> DimensionScore { - if has_first_then(lower_text) || has_step_digit(lower_text) || has_numbered_list(lower_text) { - return DimensionScore { - name: "multiStepPatterns", - score: 0.5, - signal: Some("multi-step".to_string()), - }; - } - DimensionScore::zero("multiStepPatterns") -} - -// Match the regex `first.*then`: substring "first" appears somewhere -// before substring "then". -fn has_first_then(lower_text: &str) -> bool { - match lower_text.find("first") { - Some(idx) => lower_text[idx..].contains("then"), - None => false, - } -} - -// Match the regex `step \d`: substring "step " followed by an ASCII digit. -fn has_step_digit(lower_text: &str) -> bool { - let bytes = lower_text.as_bytes(); - bytes - .windows(6) - .any(|window| &window[..5] == b"step " && window[5].is_ascii_digit()) -} - -// Match a tightened numbered-list pattern: ASCII digit, '.', whitespace, -// then an ASCII letter. Phase C retune — the original `\d\.\s` window -// matched version strings ("python 3.11 install"), floating-point -// numbers ("0.5 "), and shell prompts, firing on ~85% of calibration -// prompts. Requiring an ASCII letter after the whitespace catches actual -// list items ("1. Install ...") while ignoring numeric contexts. -fn has_numbered_list(lower_text: &str) -> bool { - let bytes = lower_text.as_bytes(); - bytes.windows(4).any(|window| { - window[0].is_ascii_digit() - && window[1] == b'.' - && window[2].is_ascii_whitespace() - && window[3].is_ascii_alphabetic() - }) -} - -/// Score `questionComplexity` — count of `?` characters in the prompt. -/// -/// `>3` questions → score `0.5` (compound multi-part request); else -/// `0.0`. Mirrors `scoreQuestionComplexity` in `rules.ts`. Caller -/// passes the raw prompt (case doesn't matter for `?`). -pub fn score_question_complexity(prompt: &str) -> DimensionScore { - let count = prompt.matches('?').count(); - if count > 3 { - return DimensionScore { - name: "questionComplexity", - score: 0.5, - signal: Some(format!("{count} questions")), - }; - } - DimensionScore::zero("questionComplexity") -} - -// Phase D: the `score_agentic_task` scorer and the -// `agentic_score` scalar were removed entirely after calibration showed -// the dimension saturating at ~72% even after retuning the keyword list -// to irreversibility markers. No keyword choice discriminated agentic -// turns from "request running inside an agent harness" on real TBLite -// traffic. Downstream estimators that needed an explicit agentic -// signal must rely on context (request shape, tool-call counts) instead. - -/// Inputs to the architecture-settled scorer. -/// -fn preview_matches(matches: &[&str]) -> String { - matches - .iter() - .take(3) - .copied() - .collect::>() - .join(", ") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn token_count_below_short_scores_negative_one() { - let result = score_token_count(20, 50, 500); - assert_eq!(result.score, -1.0); - assert_eq!(result.name, "tokenCount"); - assert!(result.signal.as_deref().unwrap().contains("short")); - } - - #[test] - fn token_count_above_long_scores_positive_one() { - let result = score_token_count(900, 50, 500); - assert_eq!(result.score, 1.0); - assert!(result.signal.as_deref().unwrap().contains("long")); - } - - #[test] - fn token_count_in_band_scores_zero() { - let result = score_token_count(200, 50, 500); - assert_eq!(result.score, 0.0); - assert!(result.signal.is_none()); - } - - #[test] - fn keyword_match_counts_substrings_with_low_high_thresholds() { - let keywords: Vec = ["foo", "bar", "baz"] - .iter() - .map(|s| s.to_string()) - .collect(); - - let none = score_keyword_match( - "nothing relevant here", - &keywords, - "x", - "lbl", - 1, - 2, - 0.5, - 1.0, - ); - assert_eq!(none.score, 0.0); - - let one = score_keyword_match( - "this mentions foo only", - &keywords, - "x", - "lbl", - 1, - 2, - 0.5, - 1.0, - ); - assert_eq!(one.score, 0.5); - - let three = - score_keyword_match("foo and bar and baz", &keywords, "x", "lbl", 1, 2, 0.5, 1.0); - assert_eq!(three.score, 1.0); - } - - #[test] - fn code_presence_fires_on_two_matches() { - let kws: Vec = ["def", "class", "function"] - .iter() - .map(|s| s.to_string()) - .collect(); - let result = score_code_presence("def foo():\n class Bar: ...", &kws); - assert_eq!(result.score, 1.0); - assert_eq!(result.name, "codePresence"); - } - - fn lowered(words: &[&str]) -> Vec { - words.iter().map(|w| w.to_lowercase()).collect() - } - - #[test] - fn reasoning_markers_use_high_low_score_band() { - let kws = lowered(&["prove", "derive", "theorem"]); - assert_eq!(score_reasoning_markers("nothing here", &kws).score, 0.0); - assert_eq!( - score_reasoning_markers("prove the theorem", &kws).score, - 1.0 - ); - assert_eq!(score_reasoning_markers("just prove it", &kws).score, 0.7); - } - - #[test] - fn technical_terms_require_two_matches_to_fire() { - let kws = lowered(&["kubernetes", "distributed", "algorithm", "protocol"]); - // single hit → 0 because low=2 - assert_eq!( - score_technical_terms("a kubernetes question", &kws).score, - 0.0 - ); - assert_eq!( - score_technical_terms("kubernetes and distributed systems", &kws).score, - 0.5 - ); - assert_eq!( - score_technical_terms("kubernetes, distributed algorithm, protocol design", &kws).score, - 1.0 - ); - } - - #[test] - fn creative_markers_capped_below_code_ceiling() { - let kws = lowered(&["story", "poem", "brainstorm"]); - assert_eq!( - score_creative_markers("write a short story", &kws).score, - 0.5 - ); - assert_eq!( - score_creative_markers("write a story and a poem", &kws).score, - 0.7 - ); - } - - #[test] - fn simple_indicators_always_negative() { - let kws = lowered(&["hello", "what is", "define"]); - assert_eq!(score_simple_indicators("hello world", &kws).score, -1.0); - assert_eq!( - score_simple_indicators("hello, define quark", &kws).score, - -1.0 - ); - assert_eq!(score_simple_indicators("hard problem", &kws).score, 0.0); - } - - #[test] - fn imperative_verbs_are_mild_signal() { - let kws = lowered(&["implement", "design", "refactor", "architect"]); - // Phase B thresholds: (low=2, high=3). One match no longer fires. - assert_eq!(score_imperative_verbs("implement a tree", &kws).score, 0.0); - assert_eq!( - score_imperative_verbs("implement and design things", &kws).score, - 0.3 - ); - assert_eq!( - score_imperative_verbs("implement, design, refactor", &kws).score, - 0.5 - ); - } - - #[test] - fn constraint_count_high_requires_three_hits() { - let kws = lowered(&["at most", "within", "no more than"]); - assert_eq!( - score_constraint_count("solve at most quickly", &kws).score, - 0.3 - ); - assert_eq!( - score_constraint_count("at most within no more than", &kws).score, - 0.7 - ); - } - - #[test] - fn output_format_signal_for_structured_keywords() { - let kws = lowered(&["json", "yaml", "csv"]); - assert_eq!(score_output_format("return json", &kws).score, 0.4); - assert_eq!(score_output_format("return json or yaml", &kws).score, 0.7); - } - - #[test] - fn reference_complexity_picks_up_context_pointers() { - let kws = lowered(&["the code above", "the api docs"]); - assert_eq!( - score_reference_complexity("use the code above", &kws).score, - 0.3 - ); - assert_eq!( - score_reference_complexity("the code above and the api docs", &kws).score, - 0.5 - ); - } - - #[test] - fn negation_complexity_requires_two_negations() { - let kws = lowered(&["not", "never", "except", "unless"]); - assert_eq!( - score_negation_complexity("this is not relevant", &kws).score, - 0.0 - ); - assert_eq!( - score_negation_complexity("not now, never tomorrow", &kws).score, - 0.3 - ); - assert_eq!( - score_negation_complexity("not, never, except", &kws).score, - 0.5 - ); - } - - #[test] - fn domain_specificity_strongest_single_keyword_signal() { - let kws = lowered(&["quantum", "fpga", "genomics"]); - assert_eq!( - score_domain_specificity("quantum question", &kws).score, - 0.5 - ); - assert_eq!( - score_domain_specificity("quantum and fpga design", &kws).score, - 0.8 - ); - } - - #[test] - fn multi_step_patterns_match_first_then_step_or_numbered_list() { - assert_eq!( - score_multi_step_patterns("nothing structured here").score, - 0.0 - ); - assert_eq!( - score_multi_step_patterns("first do x, then do y").score, - 0.5 - ); - assert_eq!( - score_multi_step_patterns("follow step 1 carefully").score, - 0.5 - ); - assert_eq!( - score_multi_step_patterns("1. install\n2. configure").score, - 0.5 - ); - } - - #[test] - fn question_complexity_fires_above_three_questions() { - assert_eq!(score_question_complexity("Is this fast?").score, 0.0); - assert_eq!(score_question_complexity("A? B? C? D? E?").score, 0.5); - } -} diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index b00a69c0..bf5441c2 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -1,22 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Tool-result context signals — thin adapter over libsy's extractor. +//! Tool-result context signals extracted from the conversation history. //! -//! The extraction logic lives in [`switchyard_libsy::ToolSignals`]. This module -//! bridges the crate's [`ChatRequest`] (a format-tagged JSON body) to libsy's -//! [`switchyard_protocol::Request`] (raw body + wire-format metadata) so the two -//! request models share one implementation. - -use std::collections::HashMap; +//! The [`DimensionCollector`][`crate::DimensionCollector`] runs this +//! alongside the 15 prompt-text scorers: it walks the `messages[]` / +//! `input[]` array of the incoming request, finds tool-execution results, +//! pattern-matches their text against a curated error table, and aggregates +//! conversation-history metrics that the stage_router pickers need. +//! +//! All logic is pure and deterministic — no I/O, no shared state. use serde_json::Value; use switchyard_core::{ChatRequest, ChatRequestType}; -use switchyard_protocol::{Metadata, Request, WireFormat}; -/// The tool-signal output type. Re-exported from libsy so downstream consumers -/// (the `ToolResultSignal` stamped on `ProxyContext`) see a single type. -pub use switchyard_libsy::{ToolSignals as ToolResultSignal, DEFAULT_RECENT_WINDOW}; +// ─── severity constants ─────────────────────────────────────────────────────── const SOFT: f32 = 0.3; const HARD: f32 = 0.7; @@ -195,8 +193,6 @@ pub struct ToolResultSignal { /// Windowed so an error persists through the recovery turns instead of clearing /// the instant the next result is clean. pub severity: f32, - /// Error pattern names from the highest-severity result in the recent window. - pub patterns: Vec, /// Consecutive clean tool results back from the most recent. `0` if the last failed. pub no_error_streak: u32, pub edit_count: u32, @@ -223,13 +219,6 @@ pub struct ToolResultSignal { /// tool results into fewer messages than OpenAI-chat), so gates keyed on it are /// approximate across request origins. pub turn_depth: u32, - /// Char count of the last user message. - pub prompt_char_count: u32, - /// Over the recent window, the largest share taken by a single identical - /// command-bearing tool call (Bash name + command), in `[0, 1]`. A weak model - /// looping the same command (e.g. `cd /tmp` many times) spikes this even when it - /// errors little and keeps "producing" — churn the error/stall signals miss. - pub repeated_cmd_ratio: f32, /// The request carries a context-compaction summary (the agent's context was /// summarised after overflowing). Compaction resets the router's accumulated /// signals, so a task that was on the strong tier de-escalates back to weak — the @@ -238,12 +227,78 @@ pub struct ToolResultSignal { pub compacted: bool, } -/// Extract tool-execution signals using the default `recent_*` window. +// `command` is the lowercased Bash command line; None for non-Bash tools. +#[derive(Debug, Clone)] +struct ObservedToolCall { + name: String, + command: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ToolCategory { + Write, + Edit, + Read, + Plan, + Other, +} + +fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { + let lower = name.to_lowercase(); + if WRITE_TOOL_NAMES.contains(&lower.as_str()) { + return ToolCategory::Write; + } + if EDIT_TOOL_NAMES.contains(&lower.as_str()) { + return ToolCategory::Edit; + } + if READ_TOOL_NAMES.contains(&lower.as_str()) { + return ToolCategory::Read; + } + if PLAN_TOOL_NAMES.contains(&lower.as_str()) { + return ToolCategory::Plan; + } + if BASH_TOOL_NAMES.contains(&lower.as_str()) { + if let Some(cmd) = command { + // Write/edit redirection trumps read-like operands. + if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { + return ToolCategory::Write; + } + if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) { + return ToolCategory::Edit; + } + if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) { + return ToolCategory::Read; + } + } + } + ToolCategory::Other +} + +// ─── extraction entry point ─────────────────────────────────────────────────── + +/// Extract all tool-execution signals from a [`ChatRequest`]. +/// +/// Uses [`DEFAULT_RECENT_WINDOW`] for the sliding-window `recent_*` counts. +/// Callers wanting a different window should use +/// [`extract_tool_signals_with_window`] directly. +/// +/// Dispatches on the request's wire format: +/// +/// * **OpenAI chat** — `messages[]` with `role: "tool"` for results; +/// `role: "assistant"` with `tool_calls[]` for call names. +/// * **Anthropic** — `messages[]` with `role: "user"` + `content[].type: +/// "tool_result"`; `role: "assistant"` with `content[].type: "tool_use"`. +/// * **OpenAI responses** — `input[]` with `type: "function_call_output"`; +/// `type: "function_call"` for call names. +/// +/// Returns [`ToolResultSignal::default()`] when the body is absent or +/// the messages list is empty — callers can always read `signal.severity`. pub fn extract_tool_signals(request: &ChatRequest) -> ToolResultSignal { - ToolResultSignal::from_request(&to_protocol_request(request), None) + extract_tool_signals_with_window(request, DEFAULT_RECENT_WINDOW) } -/// Extract tool-execution signals with a caller-supplied `recent_*` window. +/// Like [`extract_tool_signals`] but with a caller-supplied sliding-window +/// size for the `recent_*` counts. pub fn extract_tool_signals_with_window( request: &ChatRequest, recent_window: usize, @@ -319,7 +374,6 @@ fn content_has_compaction_marker(content: Option<&Value>) -> bool { fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolResultSignal { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -360,26 +414,16 @@ fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) - } } } - "user" => { - prompt_char_count = user_message_char_count(obj.get("content")); - } _ => {} } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolResultSignal { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -389,31 +433,14 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> match role { "user" => { if let Some(Value::Array(blocks)) = content { - let mut saw_text = false; for block in blocks { let Some(b) = block.as_object() else { continue }; - match b.get("type").and_then(Value::as_str) { - Some("tool_result") => { - if let Some(text) = content_to_text(b.get("content")) { - tool_texts.push(text); - } + if b.get("type").and_then(Value::as_str) == Some("tool_result") { + if let Some(text) = content_to_text(b.get("content")) { + tool_texts.push(text); } - Some("text") => { - if let Some(t) = b.get("text").and_then(Value::as_str) { - prompt_char_count = t.len() as u32; - saw_text = true; - } - } - _ => {} } } - if !saw_text { - if let Some(text_val) = content { - prompt_char_count = user_message_char_count(Some(text_val)); - } - } - } else { - prompt_char_count = user_message_char_count(content); } } "assistant" => { @@ -443,26 +470,18 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolResultSignal { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for item in items { let Some(obj) = item.as_object() else { continue; }; let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); match item_type { "function_call_output" => { @@ -488,21 +507,11 @@ fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolRe command, }); } - _ => { - if role == "user" { - prompt_char_count = user_message_char_count(obj.get("content")); - } - } + _ => {} } } - build_signal( - tool_texts, - tool_calls, - items.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, items.len() as u32, recent_window) } // ─── aggregation ───────────────────────────────────────────────────────────── @@ -511,23 +520,19 @@ fn build_signal( tool_texts: Vec, tool_calls: Vec, turn_depth: u32, - prompt_char_count: u32, recent_window: usize, ) -> ToolResultSignal { // Windowed severity: take the MAX severity across the last `recent_window` tool // results rather than only the last one. An error's severity then persists for // the recent window and decays out of it — parallel to the windowed `recent_*` - // counts and `stuck_exploring` — so a fix written a couple of turns after an error - // still routes on the error signal instead of the router flapping straight back to - // the weak tier. `patterns` carries the labels of the highest-severity result. + // counts — so a fix written a couple of turns after an error still routes on the + // error signal instead of the router flapping straight back to the weak tier. let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); let mut severity = 0.0f32; - let mut patterns: Vec = Vec::new(); for text in &tool_texts[sev_start..] { - let (sev, pats) = classify_text(text); + let (sev, _patterns) = classify_text(text); if sev > severity { severity = sev; - patterns = pats; } } @@ -587,26 +592,8 @@ fn build_signal( let tests_passed = detect_tests_passed(&tool_texts, recent_window); - // repeated_cmd_ratio: over the recent window, the largest count of an identical - // command-bearing call (Bash name + command) / window. Catches a weak model - // looping the same command with no progress — the churn that severity/spinning - // miss because it errors little and still "produces". Only command-bearing calls - // count, so distinct reads/writes don't false-positive. - let mut cmd_counts: HashMap<(&str, &str), u32> = HashMap::new(); - for tc in tool_calls.iter().skip(recent_start) { - if let Some(cmd) = tc.command.as_deref() { - *cmd_counts.entry((tc.name.as_str(), cmd)).or_insert(0) += 1; - } - } - let repeated_cmd_ratio = if recent_window > 0 { - cmd_counts.values().copied().max().unwrap_or(0) as f32 / recent_window as f32 - } else { - 0.0 - }; - ToolResultSignal { severity, - patterns, no_error_streak, edit_count, write_count, @@ -619,8 +606,6 @@ fn build_signal( pure_bash_streak, tests_passed, turn_depth, - prompt_char_count, - repeated_cmd_ratio, // Set by extract_tool_signals_with_window after the format-specific extract, // which scans all message contents for the compaction marker. compacted: false, @@ -653,24 +638,6 @@ fn content_to_text(content: Option<&Value>) -> Option { } } -/// Character count of a user-message content value. -fn user_message_char_count(content: Option<&Value>) -> u32 { - match content { - Some(Value::String(s)) => s.len() as u32, - Some(Value::Array(blocks)) => blocks - .iter() - .filter_map(|b| { - b.as_object() - .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - }) - .map(|s| s.len() as u32) - .sum(), - _ => 0, - } -} - /// Match `text` against the error pattern table. /// /// Returns `(max_severity, matched_pattern_names)`. @@ -855,7 +822,6 @@ mod tests { })); let sig = extract_tool_signals(&request); assert_eq!(sig.severity, HARD); - assert!(sig.patterns.contains(&"traceback".to_string())); assert_eq!(sig.edit_count, 1); assert_eq!(sig.turn_depth, 3); } @@ -945,24 +911,6 @@ mod tests { assert_eq!(wide.recent_edit_count, 1); } - #[test] - fn repeated_cmd_ratio_spikes_on_a_looping_command() { - // Five identical `cd /tmp` calls fill a window of 5 → ratio 1.0 (churn). - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"cd /tmp\"}"}}]}, - ] - })); - assert_eq!( - extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, - 1.0 - ); - } - #[test] fn compaction_marker_sets_compacted() { // The compaction summary is a user message carrying Claude Code's preamble. @@ -986,24 +934,6 @@ mod tests { assert!(!extract_tool_signals(&request).compacted); } - #[test] - fn repeated_cmd_ratio_stays_low_for_distinct_commands() { - // Five distinct commands → max repeat 1 / window 5 = 0.2, no false churn. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls a\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls b\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls c\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls d\"}"}}]}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls e\"}"}}]}, - ] - })); - assert_eq!( - extract_tool_signals_with_window(&request, 5).repeated_cmd_ratio, - 0.2 - ); - } - #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs index 3b34c631..094f58e5 100644 --- a/crates/switchyard-components/src/lib.rs +++ b/crates/switchyard-components/src/lib.rs @@ -12,6 +12,7 @@ pub mod dimension_collector; pub mod intake; pub mod request_processors; pub mod response_processors; +pub mod stage_router; pub mod stats; mod telemetry; @@ -20,8 +21,7 @@ pub use backends::{ MultiLlmBackend, OpenAiNativeBackend, OpenAiPassthroughBackend, StatsLlmBackend, }; pub use dimension_collector::{ - extract_tool_signals, ContextSignals, DimensionScore, Keywords, ResponseFlag, ResponseSignals, - ScoringConfig, ToolResultSignal, + extract_tool_signals, ResponseFlag, ResponseSignals, ToolResultSignal, }; pub use intake::{ HttpIntakeSink, IntakeFormat, IntakePayloadBuilder, IntakeQueueFullPolicy, diff --git a/crates/switchyard-components/src/request_processors/dimension_collector.rs b/crates/switchyard-components/src/request_processors/dimension_collector.rs index f5558c62..617ae1a7 100644 --- a/crates/switchyard-components/src/request_processors/dimension_collector.rs +++ b/crates/switchyard-components/src/request_processors/dimension_collector.rs @@ -1,63 +1,40 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Request-processor adapter for the context-signal extractor. +//! Request-processor adapter for the tool-signal extractor. //! -//! Thin wrapper around [`crate::dimension_collector::extract_signals`]. -//! The pure logic lives in [`crate::dimension_collector`]; this file is -//! the chain entry point — it pulls the user prompt out of `ChatRequest`, -//! lowercases it once, hands off to the extractor, and stamps the -//! resulting [`ContextSignals`] into `ProxyContext` via the typed -//! extensions bag. +//! Thin wrapper around [`crate::dimension_collector::extract_tool_signals_with_window`]. +//! It walks the request's tool-call history and stamps the resulting +//! [`ToolResultSignal`] into `ProxyContext` for the stage_router picker to read. -use serde_json::Value; use switchyard_core::{ChatRequest, ProxyContext, Result}; use crate::dimension_collector::{ - extract_signals, extract_tool_signals_with_window, ContextSignals, ScoringConfig, - ToolResultSignal, DEFAULT_RECENT_WINDOW, + extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, }; -/// Populates `ProxyContext` with scored dimensions extracted from the prompt. -/// -/// Read by downstream estimators (LLM classifier, future rules -/// estimator) via `ctx.get::()`. +/// Populates `ProxyContext` with a [`ToolResultSignal`] read from the request's +/// tool-call history. The stage_router picker reads it via +/// `ctx.get::()`. #[derive(Clone, Debug)] pub struct DimensionCollector { - config: ScoringConfig, recent_window: usize, } impl Default for DimensionCollector { fn default() -> Self { Self { - config: ScoringConfig::default(), recent_window: DEFAULT_RECENT_WINDOW, } } } impl DimensionCollector { - /// Construct a collector with an explicit scoring config and the default - /// `recent_*` window size ([`DEFAULT_RECENT_WINDOW`]). - pub fn new(config: ScoringConfig) -> Self { - Self::with_recent_window(config, DEFAULT_RECENT_WINDOW) - } - - /// Construct a collector with a caller-supplied sliding-window size for - /// `recent_*` signal counts. Smaller windows make the picker more - /// reactive to the very last tool call; larger windows smooth over - /// noisy turn-by-turn fluctuations. - pub fn with_recent_window(config: ScoringConfig, recent_window: usize) -> Self { - Self { - config, - recent_window, - } - } - - /// Returns the underlying scoring config (for tests / introspection). - pub fn config(&self) -> &ScoringConfig { - &self.config + /// Construct a collector with a caller-supplied sliding-window size for the + /// `recent_*` signal counts. Smaller windows make the picker more reactive + /// to the latest tool call; larger windows smooth over turn-by-turn noise. + pub fn with_recent_window(recent_window: usize) -> Self { + Self { recent_window } } /// Returns the configured `recent_*` sliding-window size. @@ -65,132 +42,29 @@ impl DimensionCollector { self.recent_window } - /// Extracts request-side signals and stores them on the request context. + /// Extracts the tool-result signal and stores it on the request context. pub async fn process( &self, ctx: &mut ProxyContext, request: ChatRequest, ) -> Result { - // Text-dimension pass (existing 15 scorers). - let prompt = extract_user_prompt(&request).unwrap_or_default(); - let lower = prompt.to_lowercase(); - let signals = extract_signals(&prompt, &lower, &self.config); - ctx.insert::(signals); - // Tool-signal pass: walk the messages array for tool results and call names. let tool_signal = extract_tool_signals_with_window(&request, self.recent_window); ctx.insert::(tool_signal); Ok(request) } } -/// Extract a single concatenated user-text blob from any inbound `ChatRequest`. -/// -/// Looks at the request body's `messages` array (OpenAI chat / Anthropic -/// messages) or `input` field (OpenAI responses) and concatenates all -/// string user content. Tool calls, role-system messages, and structured -/// content blocks are intentionally ignored — they're either non-user -/// signal or handled by the upcoming dedicated scorers. -fn extract_user_prompt(request: &ChatRequest) -> Option { - let body = request.body(); - let object = body.as_object()?; - - if let Some(messages) = object.get("messages").and_then(Value::as_array) { - let collected = collect_user_text_from_messages(messages); - if !collected.is_empty() { - return Some(collected); - } - } - - if let Some(input) = object.get("input") { - match input { - Value::String(text) => return Some(text.clone()), - Value::Array(items) => { - let collected = collect_user_text_from_messages(items); - if !collected.is_empty() { - return Some(collected); - } - } - _ => {} - } - } - - None -} - -fn collect_user_text_from_messages(messages: &[Value]) -> String { - let mut chunks: Vec = Vec::new(); - for message in messages { - let Some(object) = message.as_object() else { - continue; - }; - let role = object.get("role").and_then(Value::as_str).unwrap_or(""); - if role != "user" { - continue; - } - match object.get("content") { - Some(Value::String(text)) => chunks.push(text.clone()), - Some(Value::Array(parts)) => { - for part in parts { - if let Some(text) = part - .as_object() - .and_then(|p| p.get("text")) - .and_then(Value::as_str) - { - chunks.push(text.to_string()); - } - } - } - _ => {} - } - } - chunks.join("\n") -} - #[cfg(test)] mod tests { use super::*; - use crate::dimension_collector::Keywords; use serde_json::json; #[tokio::test] - async fn stamps_context_signals_into_proxy_context() { - let collector = DimensionCollector::new(ScoringConfig { - code_keywords: Keywords::new(["def", "class"]), - ..ScoringConfig::default() - }); - - let request = ChatRequest::openai_chat(json!({ - "model": "test-model", - "messages": [ - {"role": "system", "content": "you are a helpful assistant"}, - {"role": "user", "content": "def hello(): class Foo: pass"}, - ], - })); - - let mut ctx = ProxyContext::new(); - let _ = collector - .process(&mut ctx, request) - .await - .expect("process ok"); - - let signals = ctx - .get::() - .expect("ContextSignals stamped into context"); - assert_eq!(signals.dimensions.len(), 14); - let code_presence = signals - .dimensions - .iter() - .find(|dim| dim.name == "codePresence") - .expect("codePresence dimension present"); - assert_eq!(code_presence.score, 1.0); - } - - #[tokio::test] - async fn handles_request_with_no_user_content_gracefully() { + async fn stamps_tool_result_signal_into_proxy_context() { let collector = DimensionCollector::default(); let request = ChatRequest::openai_chat(json!({ "model": "test-model", - "messages": [{"role": "system", "content": "only system"}], + "messages": [{"role": "user", "content": "hi"}], })); let mut ctx = ProxyContext::new(); @@ -199,8 +73,6 @@ mod tests { .await .expect("process ok"); - let signals = ctx.get::().expect("signals stamped"); - // Empty user prompt → 0 estimated tokens → tokenCount fires as "short". - assert_eq!(signals.token_count_estimate, 0); + assert!(ctx.get::().is_some()); } } diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs new file mode 100644 index 00000000..f8171137 --- /dev/null +++ b/crates/switchyard-components/src/stage_router.rs @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Stage-router scoring and tier selection — the shared routing core. +//! +//! Given a [`ToolResultSignal`] (extracted by the dimension collector), this +//! module decides whether a coding-agent turn should go to the **capable** +//! (strong) or **efficient** (weak) tier. It is the single source of truth for +//! the decision, shared by the Rust profile and the Python processor via +//! bindings — only the outer shell differs in how it fetches the decision. +//! +//! Two axes: +//! +//! * **error** — did the recent tool results error? (`severity`) +//! * **production** — is the agent producing code? (`spinning` / `exploring` +//! push toward capable; `production_intensity` pushes toward efficient) +//! +//! Signals are scored with fixed weights, summed, and `tanh`-squashed into +//! `(-1, +1)`; `confidence` is the magnitude. The `confidence_threshold` dials +//! how much corroboration a decisive escalation needs (see [`score_signal`]). + +use crate::dimension_collector::ToolResultSignal; + +/// Turn depth below which stall signals stay quiet — early no-write turns are +/// normal exploration, not a stall. +const STALL_MIN_TURN_DEPTH: u32 = 8; +/// Gain applied before the tanh squash — spreads the small raw weighted sum +/// across the usable confidence range. Without it confidence would cap near +/// ±0.20 and mid/high thresholds would be unreachable. +const SCORE_GAIN: f64 = 5.0; +/// Strongest error severity the scorer sees: critical (`1.0`) is caught by the +/// override, so hard (`0.7`) normalises `severity` to one signal unit. +const HARD_SEVERITY: f64 = 0.7; +/// Weight one maxed signal contributes. Small enough that no single axis pegs +/// the decision; corroboration across the two axes is what raises confidence. +const SIGNAL_UNIT: f64 = 0.10; +/// Critical severity forces the capable tier regardless of the scorer. +const SEVERITY_CRITICAL: f32 = 1.0; + +/// Signed, fixed weights over the two axes. Error signals (`severity`, +/// `spinning`, `exploring`) push toward capable (+); `production_intensity` +/// pushes toward efficient (−). `severity` is normalised by its hard cap so it +/// too contributes one `SIGNAL_UNIT` when maxed. +const DEFAULT_WEIGHTS: &[(&str, f64)] = &[ + ("severity", SIGNAL_UNIT / HARD_SEVERITY), + ("spinning", SIGNAL_UNIT), + ("exploring", SIGNAL_UNIT), + ("production_intensity", -SIGNAL_UNIT), +]; + +/// The two tiers a turn can route to. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Tier { + /// Weak / cheap tier. + Efficient, + /// Strong / capable tier. + Capable, +} + +/// Which tier to default to when the scorer is not confident. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PickerMode { + /// Default to capable unless the scorer confidently picks efficient. + CapableFirst, + /// Default to efficient unless the scorer confidently picks capable. + EfficientFirst, +} + +impl PickerMode { + fn default_tier(self) -> Tier { + match self { + Self::CapableFirst => Tier::Capable, + Self::EfficientFirst => Tier::Efficient, + } + } +} + +/// What produced a decision — for stats and explainability. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DecisionSource { + /// Hard override (critical severity or context compaction). + Override, + /// Settled run: recent tests passed with recent production and no error. + TestsPassed, + /// Scorer crossed `confidence_threshold`. + Dimensions, + /// Scorer was not confident; the caller should consult its classifier. + FallOpen, +} + +impl DecisionSource { + /// Stable lowercase label used in stats. + pub fn as_str(self) -> &'static str { + match self { + Self::Override => "override", + Self::TestsPassed => "tests_passed", + Self::Dimensions => "dimensions", + Self::FallOpen => "fall_open", + } + } +} + +/// A signed score in `(-1, +1)` and its magnitude. `confidence == score.abs()`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ScoreResult { + /// Signed score: positive → capable, negative → efficient. + pub score: f64, + /// Decision certainty, `score.abs()`. + pub confidence: f64, +} + +/// The two-axis feature view of a single [`ToolResultSignal`]. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CodingAgentDimensions { + /// Windowed max error severity in `[0, 1]`. + pub severity: f64, + /// `1.0` when a deep turn has no reads, plans, writes, or edits (pure churn). + pub spinning: f64, + /// `1.0` when a deep turn reads/plans but does not write or edit. + pub exploring: f64, + /// Fraction of recent tool ops that produced code (writes + edits). + pub production_intensity: f64, +} + +impl CodingAgentDimensions { + fn value(&self, name: &str) -> f64 { + match name { + "severity" => self.severity, + "spinning" => self.spinning, + "exploring" => self.exploring, + "production_intensity" => self.production_intensity, + _ => 0.0, + } + } +} + +/// Outcome of [`pick_tier`]: either a resolved decision, or a signal that the +/// caller should consult its (impl-specific, async) classifier. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PickOutcome { + /// The tier was decided without the classifier. + Resolved { + /// The chosen tier. + tier: Tier, + /// What produced it. + source: DecisionSource, + /// Signed scorer value (`0.0` for override / tests-passed). + score: f64, + /// Scorer confidence (`None` where the scorer did not run). + confidence: Option, + }, + /// The scorer was below threshold. The caller runs its classifier; if it + /// has none (or it fails), fall open to `default_tier`. + ConsultClassifier { + /// Signed scorer value, for logging. + score: f64, + /// Scorer confidence, for logging. + confidence: f64, + /// Tier to fall open to when no classifier resolves it. + default_tier: Tier, + }, +} + +/// Project a [`ToolResultSignal`] onto the two-axis dimension space. +pub fn dimensions_from_signal(signal: &ToolResultSignal) -> CodingAgentDimensions { + let recent_ops = signal.recent_write_count + + signal.recent_edit_count + + signal.recent_read_count + + signal.recent_todowrite_count; + let deep_enough = signal.turn_depth >= STALL_MIN_TURN_DEPTH; + let no_production = signal.recent_write_count == 0 && signal.recent_edit_count == 0; + let investigating = signal.recent_read_count >= 1 || signal.recent_todowrite_count >= 1; + // spinning vs exploring partition the "not producing" case by investigative + // activity, so at most one fires — no double-counting on the production axis. + let spinning = deep_enough && no_production && !investigating; + let exploring = deep_enough && no_production && investigating; + + CodingAgentDimensions { + severity: f64::from(signal.severity), + spinning: if spinning { 1.0 } else { 0.0 }, + exploring: if exploring { 1.0 } else { 0.0 }, + production_intensity: ratio( + signal.recent_write_count + signal.recent_edit_count, + recent_ops, + ), + } +} + +/// Score a signal: weighted sum of the dimensions, `tanh`-squashed. +/// +/// The raw sum is small — one maxed signal is `±0.10`, two corroborating +/// signals `±0.20`. `tanh(gain·raw)` spreads that into a usable range, so the +/// `confidence_threshold` reads roughly: `~0.3` escalates on one signal, `~0.5` +/// needs about one-and-a-half, `~0.7` needs two to corroborate. +pub fn score_signal(signal: &ToolResultSignal) -> ScoreResult { + let dimensions = dimensions_from_signal(signal); + let raw: f64 = DEFAULT_WEIGHTS + .iter() + .map(|(name, weight)| dimensions.value(name) * weight) + .sum(); + let score = (SCORE_GAIN * raw).tanh(); + ScoreResult { + score, + confidence: score.abs(), + } +} + +/// Hard **escalate** — force the strong tier no matter what the scorer would +/// say. Fires on a critical error or a compacted context. +fn should_escalate(signal: &ToolResultSignal) -> bool { + // Compaction wipes the accumulated signals, so a task that had escalated + // would snap back to weak — a context big enough to overflow belongs strong. + if signal.compacted { + return true; + } + // A critical error is unambiguous. + signal.severity >= SEVERITY_CRITICAL +} + +/// Hard **de-escalate** — drop to the cheap tier on a settled turn: tests +/// passed, code was just written or edited, and nothing errored in the window. +fn should_deescalate(signal: &ToolResultSignal) -> bool { + signal.tests_passed + && (signal.recent_write_count + signal.recent_edit_count) >= 1 + && signal.severity <= 0.0 +} + +/// Decide a turn's tier from its signal. +/// +/// The rules run in order; the first that fires wins: +/// +/// 1. **Escalate** — a hard reason to go strong (critical error / compaction). +/// 2. **De-escalate** — a hard reason to go cheap (a settled turn). +/// 3. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it. +/// 4. **Fall open** — not confident: hand to the classifier, else the default. +/// +/// Rules 1 and 2 are the two hard shortcuts that skip the scorer — one always +/// escalates, one always de-escalates. **Escalate is checked first**, so a +/// critical error still wins on a turn whose tests also happened to pass. +/// +/// Deterministic and pure: the async classifier lives in the caller, so rule 4 +/// returns [`PickOutcome::ConsultClassifier`] instead of calling it here. The +/// `no_signal` case (no tool activity yet) is handled one level up. +pub fn pick_tier( + signal: &ToolResultSignal, + mode: PickerMode, + confidence_threshold: f64, +) -> PickOutcome { + // 1. Escalate — a hard reason to go strong, ahead of everything else. + if should_escalate(signal) { + return resolved(Tier::Capable, DecisionSource::Override, 0.0, Some(1.0)); + } + + // 2. De-escalate — a hard reason to go cheap (the turn is winding down). + if should_deescalate(signal) { + return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.0, None); + } + + // 3. Scorer — no hard reason either way, so weigh error vs production. If + // confident enough, follow the sign: positive → strong, negative → cheap. + let scored = score_signal(signal); + if scored.confidence >= confidence_threshold { + let tier = if scored.score > 0.0 { + Tier::Capable + } else { + Tier::Efficient + }; + return resolved(tier, DecisionSource::Dimensions, scored.score, Some(scored.confidence)); + } + + // 4. Fall open — the signals didn't corroborate enough to be sure. Hand off + // to the caller's classifier; with none, land on the picker's default. + PickOutcome::ConsultClassifier { + score: scored.score, + confidence: scored.confidence, + default_tier: mode.default_tier(), + } +} + +/// Build a resolved outcome (a decision made without the classifier). +fn resolved(tier: Tier, source: DecisionSource, score: f64, confidence: Option) -> PickOutcome { + PickOutcome::Resolved { + tier, + source, + score, + confidence, + } +} + +fn ratio(numerator: u32, denominator: u32) -> f64 { + if denominator == 0 { + 0.0 + } else { + f64::from(numerator) / f64::from(denominator) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dimension_collector::extract_tool_signals; + use switchyard_core::ChatRequest; + use serde_json::json; + + fn signal_from(messages: serde_json::Value) -> ToolResultSignal { + let request = ChatRequest::openai_chat(json!({"model": "m", "messages": messages})); + extract_tool_signals(&request) + } + + #[test] + fn critical_severity_overrides_to_capable() { + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = SEVERITY_CRITICAL; + match pick_tier(&signal, PickerMode::EfficientFirst, 0.5) { + PickOutcome::Resolved { tier, source, .. } => { + assert_eq!(tier, Tier::Capable); + assert_eq!(source, DecisionSource::Override); + } + other => panic!("expected override, got {other:?}"), + } + } + + #[test] + fn compaction_overrides_to_capable() { + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.compacted = true; + assert!(matches!( + pick_tier(&signal, PickerMode::EfficientFirst, 0.5), + PickOutcome::Resolved { tier: Tier::Capable, source: DecisionSource::Override, .. } + )); + } + + #[test] + fn one_signal_scores_below_half() { + // A single full wrong signal ≈ 0.46 confidence — just under 0.5. + let mut signal = signal_from(json!([{"role": "user", "content": "hi"}])); + signal.severity = HARD_SEVERITY as f32; + let scored = score_signal(&signal); + assert!(scored.score > 0.0); + assert!(scored.confidence < 0.5, "one signal should not clear 0.5: {scored:?}"); + } + + #[test] + fn quiet_signal_falls_open_to_default() { + let signal = signal_from(json!([{"role": "user", "content": "hi"}])); + match pick_tier(&signal, PickerMode::EfficientFirst, 0.5) { + PickOutcome::ConsultClassifier { default_tier, .. } => { + assert_eq!(default_tier, Tier::Efficient); + } + other => panic!("expected consult-classifier, got {other:?}"), + } + } +} diff --git a/crates/switchyard-py/src/component_bindings.rs b/crates/switchyard-py/src/component_bindings.rs index 9a2c7202..5965811c 100644 --- a/crates/switchyard-py/src/component_bindings.rs +++ b/crates/switchyard-py/src/component_bindings.rs @@ -11,6 +11,7 @@ mod dimension_collector; pub(crate) mod intake; mod request_processors; mod response_processors; +mod stage_router; pub(crate) mod stats; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,5 +22,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { response_processors::register(module)?; backends::register(module)?; dimension_collector::register(module)?; + stage_router::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/component_bindings/dimension_collector.rs b/crates/switchyard-py/src/component_bindings/dimension_collector.rs index 5fd8539f..8e161ddb 100644 --- a/crates/switchyard-py/src/component_bindings/dimension_collector.rs +++ b/crates/switchyard-py/src/component_bindings/dimension_collector.rs @@ -19,9 +19,8 @@ use pyo3::prelude::*; use pyo3::types::PyList; use switchyard_components::{ dimension_collector::{ - extract_response_signals as core_extract_response_signals, ContextSignals, DimensionScore, - Keywords, ResponseFlag, ResponseSignals, ScoringConfig, ToolResultSignal, - DEFAULT_RECENT_WINDOW, + extract_response_signals as core_extract_response_signals, ResponseFlag, ResponseSignals, + ToolResultSignal, DEFAULT_RECENT_WINDOW, }, DimensionCollector, ResponseSignalCollector, }; @@ -33,161 +32,6 @@ use crate::core_bindings::context::PyProxyContext; use crate::core_bindings::request::PyChatRequest; use crate::core_bindings::response::PyChatResponse; -/// One scorer's output as a Python-visible record. -#[pyclass(name = "DimensionScore", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyDimensionScore { - inner: DimensionScore, -} - -#[pymethods] -impl PyDimensionScore { - #[getter] - fn name(&self) -> &'static str { - self.inner.name - } - - #[getter] - fn score(&self) -> f32 { - self.inner.score - } - - #[getter] - fn signal(&self) -> Option<&str> { - self.inner.signal.as_deref() - } - - fn __repr__(&self) -> String { - format!( - "DimensionScore(name={:?}, score={}, signal={:?})", - self.inner.name, self.inner.score, self.inner.signal, - ) - } -} - -impl PyDimensionScore { - fn from_core(inner: DimensionScore) -> Self { - Self { inner } - } -} - -/// Aggregate context-signal record stamped by [`PyDimensionCollector`]. -#[pyclass(name = "ContextSignals", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -pub(crate) struct PyContextSignals { - inner: ContextSignals, -} - -#[pymethods] -impl PyContextSignals { - /// The 15 scored dimensions in canonical order. - #[getter] - fn dimensions(&self, py: Python<'_>) -> PyResult> { - let items: Vec> = self - .inner - .dimensions - .iter() - .map(|dim| Py::new(py, PyDimensionScore::from_core(dim.clone()))) - .collect::>>()?; - Ok(PyList::new(py, items)?.unbind()) - } - - /// Estimated input-token count (`chars / 4` heuristic for now). - #[getter] - fn token_count_estimate(&self) -> u32 { - self.inner.token_count_estimate - } - - fn __repr__(&self) -> String { - format!( - "ContextSignals(dimensions=<{} items>, token_count_estimate={})", - self.inner.dimensions.len(), - self.inner.token_count_estimate, - ) - } -} - -impl PyContextSignals { - fn from_core(inner: ContextSignals) -> Self { - Self { inner } - } -} - -/// Python-facing scoring config; keyword lists are lower-cased on the way in. -#[pyclass(name = "ScoringConfig", skip_from_py_object)] -#[derive(Clone, Debug, Default)] -pub(crate) struct PyScoringConfig { - inner: ScoringConfig, -} - -#[pymethods] -impl PyScoringConfig { - #[new] - #[pyo3(signature = ( - token_count_short = 50, - token_count_long = 500, - code_keywords = vec![], - reasoning_keywords = vec![], - simple_keywords = vec![], - technical_keywords = vec![], - creative_keywords = vec![], - imperative_verbs = vec![], - constraint_indicators = vec![], - output_format_keywords = vec![], - reference_keywords = vec![], - negation_keywords = vec![], - domain_specific_keywords = vec![], - ))] - #[allow(clippy::too_many_arguments)] - fn py_new( - token_count_short: u32, - token_count_long: u32, - code_keywords: Vec, - reasoning_keywords: Vec, - simple_keywords: Vec, - technical_keywords: Vec, - creative_keywords: Vec, - imperative_verbs: Vec, - constraint_indicators: Vec, - output_format_keywords: Vec, - reference_keywords: Vec, - negation_keywords: Vec, - domain_specific_keywords: Vec, - ) -> Self { - let inner = ScoringConfig { - token_count: switchyard_components::dimension_collector::TokenCountThresholds { - short: token_count_short, - long: token_count_long, - }, - code_keywords: Keywords::new(code_keywords), - reasoning_keywords: Keywords::new(reasoning_keywords), - simple_keywords: Keywords::new(simple_keywords), - technical_keywords: Keywords::new(technical_keywords), - creative_keywords: Keywords::new(creative_keywords), - imperative_verbs: Keywords::new(imperative_verbs), - constraint_indicators: Keywords::new(constraint_indicators), - output_format_keywords: Keywords::new(output_format_keywords), - reference_keywords: Keywords::new(reference_keywords), - negation_keywords: Keywords::new(negation_keywords), - domain_specific_keywords: Keywords::new(domain_specific_keywords), - }; - Self { inner } - } - - fn __repr__(&self) -> String { - format!( - "ScoringConfig(token_count_short={}, token_count_long={})", - self.inner.token_count.short, self.inner.token_count.long, - ) - } -} - -impl PyScoringConfig { - fn clone_core(&self) -> ScoringConfig { - self.inner.clone() - } -} - /// Request-side component that runs the dimension collector. #[pyclass(name = "DimensionCollector", skip_from_py_object)] #[derive(Clone, Debug)] @@ -198,12 +42,11 @@ pub(crate) struct PyDimensionCollector { #[pymethods] impl PyDimensionCollector { #[new] - #[pyo3(signature = (config = None, *, recent_window = None))] - fn py_new(config: Option>, recent_window: Option) -> Self { - let scoring = config.map(|cfg| cfg.clone_core()).unwrap_or_default(); + #[pyo3(signature = (*, recent_window = None))] + fn py_new(recent_window: Option) -> Self { let window = recent_window.unwrap_or(DEFAULT_RECENT_WINDOW); Self { - inner: DimensionCollector::with_recent_window(scoring, window), + inner: DimensionCollector::with_recent_window(window), } } @@ -240,18 +83,6 @@ impl PyDimensionCollector { } } -/// Returns the `ContextSignals` stamped by a `DimensionCollector` run. -/// -/// Mirrors the Python idiom `ctx.metadata.get("context_signals")` from -/// the deleted Python implementation, but uses the typed extension bag -/// so consumers don't pay for the dict round-trip. -#[pyfunction] -fn get_context_signals(ctx: PyRef<'_, PyProxyContext>) -> PyResult> { - Ok(ctx - .get_cloned::()? - .map(PyContextSignals::from_core)) -} - /// Closed set of response-side quality flags emitted by the response /// signal collector. Mirrors Rust's /// [`switchyard_components::dimension_collector::ResponseFlag`] enum. @@ -451,18 +282,6 @@ impl PyToolResultSignal { self.inner.severity } - /// Pattern names that fired in the most recent tool result. - #[getter] - fn patterns(&self, py: Python<'_>) -> PyResult> { - let items: Vec> = self - .inner - .patterns - .iter() - .map(|p| Ok(pyo3::types::PyString::new(py, p).unbind())) - .collect::>()?; - Ok(PyList::new(py, items)?.unbind()) - } - /// Consecutive clean tool results at the end of history (``0`` if last failed). #[getter] fn no_error_streak(&self) -> u32 { @@ -535,19 +354,6 @@ impl PyToolResultSignal { self.inner.turn_depth } - /// Character count of the last user message (current-ask size). - #[getter] - fn prompt_char_count(&self) -> u32 { - self.inner.prompt_char_count - } - - /// Largest share taken by a single identical command-bearing tool call over the - /// recent window, in `[0, 1]` — a weak model looping the same command (churn). - #[getter] - fn repeated_cmd_ratio(&self) -> f32 { - self.inner.repeated_cmd_ratio - } - /// The request carries a context-compaction summary — the picker forces + holds /// the strong tier, since compaction otherwise de-escalates the router to weak. #[getter] @@ -571,6 +377,11 @@ impl PyToolResultSignal { fn from_core(inner: ToolResultSignal) -> Self { Self { inner } } + + /// The underlying Rust signal — used by the stage_router picker binding. + pub(crate) fn core(&self) -> &ToolResultSignal { + &self.inner + } } /// Returns the :class:`ToolResultSignal` stamped by a :class:`DimensionCollector` run. @@ -584,15 +395,11 @@ fn get_tool_result_signal(ctx: PyRef<'_, PyProxyContext>) -> PyResult) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; module.add_class::()?; - module.add_function(wrap_pyfunction!(get_context_signals, module)?)?; module.add_function(wrap_pyfunction!(get_response_signals, module)?)?; module.add_function(wrap_pyfunction!(extract_response_signals, module)?)?; module.add_function(wrap_pyfunction!(get_tool_result_signal, module)?)?; diff --git a/crates/switchyard-py/src/component_bindings/stage_router.rs b/crates/switchyard-py/src/component_bindings/stage_router.rs new file mode 100644 index 00000000..b8795e62 --- /dev/null +++ b/crates/switchyard-py/src/component_bindings/stage_router.rs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Python binding for the shared stage_router picker. +//! +//! Exposes [`switchyard_components::stage_router::pick_tier`] as +//! `stage_pick_tier(signal, picker_mode, confidence_threshold) -> PickOutcome` +//! so the Python `processor.py` runs the exact same routing decision as the Rust +//! profile. The async classifier and the `no_signal` case stay in Python: this +//! returns a resolved decision, or a request to consult the classifier. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +use switchyard_components::stage_router::{ + pick_tier, score_signal, PickOutcome, PickerMode, Tier, +}; + +use super::dimension_collector::PyToolResultSignal; + +fn parse_picker_mode(mode: &str) -> PyResult { + match mode { + "capable_first" => Ok(PickerMode::CapableFirst), + "efficient_first" => Ok(PickerMode::EfficientFirst), + other => Err(PyValueError::new_err(format!( + "unknown picker mode {other:?} (expected capable_first or efficient_first)" + ))), + } +} + +fn tier_str(tier: Tier) -> &'static str { + match tier { + Tier::Capable => "capable", + Tier::Efficient => "efficient", + } +} + +/// Result of [`stage_pick_tier`]. +/// +/// `resolved` is `True` when the picker decided without the classifier — then +/// `tier` and `source` are set. `resolved` is `False` when the scorer was not +/// confident: the caller runs its classifier, and falls back to `default_tier` +/// if it has none or it fails. `score` / `confidence` are always the scorer's. +#[pyclass(name = "PickOutcome", frozen)] +pub(crate) struct PyPickOutcome { + resolved: bool, + tier: Option<&'static str>, + source: Option<&'static str>, + default_tier: &'static str, + score: f64, + confidence: Option, +} + +#[pymethods] +impl PyPickOutcome { + #[getter] + fn resolved(&self) -> bool { + self.resolved + } + + /// Chosen tier (`"capable"` / `"efficient"`) — only when `resolved`. + #[getter] + fn tier(&self) -> Option<&'static str> { + self.tier + } + + /// Decision source (`"override"` / `"tests_passed"` / `"dimensions"`) — only + /// when `resolved`. + #[getter] + fn source(&self) -> Option<&'static str> { + self.source + } + + /// Tier to fall open to when the classifier does not resolve the turn. + #[getter] + fn default_tier(&self) -> &'static str { + self.default_tier + } + + #[getter] + fn score(&self) -> f64 { + self.score + } + + #[getter] + fn confidence(&self) -> Option { + self.confidence + } + + fn __repr__(&self) -> String { + if self.resolved { + format!( + "PickOutcome(resolved, tier={:?}, source={:?}, score={:.3})", + self.tier, self.source, self.score + ) + } else { + format!( + "PickOutcome(consult_classifier, default_tier={:?}, score={:.3})", + self.default_tier, self.score + ) + } + } +} + +/// Decide a turn's tier from its [`ToolResultSignal`], up to (but not including) +/// the classifier. `picker_mode` is `"capable_first"` or `"efficient_first"`. +#[pyfunction] +fn stage_pick_tier( + signal: PyRef<'_, PyToolResultSignal>, + picker_mode: &str, + confidence_threshold: f64, +) -> PyResult { + let mode = parse_picker_mode(picker_mode)?; + let outcome = match pick_tier(signal.core(), mode, confidence_threshold) { + PickOutcome::Resolved { + tier, + source, + score, + confidence, + } => PyPickOutcome { + resolved: true, + tier: Some(tier_str(tier)), + source: Some(source.as_str()), + default_tier: tier_str(tier), + score, + confidence, + }, + PickOutcome::ConsultClassifier { + score, + confidence, + default_tier, + } => PyPickOutcome { + resolved: false, + tier: None, + source: None, + default_tier: tier_str(default_tier), + score, + confidence: Some(confidence), + }, + }; + Ok(outcome) +} + +/// The pure two-axis scorer for a signal, as `(score, confidence)`. Used by +/// offline analysis to replay the raw score independent of the picker mode. +#[pyfunction] +fn stage_score_signal(signal: PyRef<'_, PyToolResultSignal>) -> (f64, f64) { + let result = score_signal(signal.core()); + (result.score, result.confidence) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(stage_pick_tier, module)?)?; + module.add_function(wrap_pyfunction!(stage_score_signal, module)?)?; + Ok(()) +} diff --git a/switchyard/lib/processors/stage_router/__init__.py b/switchyard/lib/processors/stage_router/__init__.py index 358633ba..2eadd2fc 100644 --- a/switchyard/lib/processors/stage_router/__init__.py +++ b/switchyard/lib/processors/stage_router/__init__.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""StageRouter — weighted scorer + selective LLM-classifier.""" +"""StageRouter — Rust two-axis scorer + picker, with a selective LLM classifier. + +The routing logic lives in Rust (``switchyard_components::stage_router``); this +package is the Python shell: the picker orchestration, the async classifier, +handoff notes, and decision logging.""" from switchyard.lib.processors.stage_router.classifier import ( CAPABLE_TIER, @@ -13,10 +17,6 @@ DecisionSource, StageRouterDecisionLog, ) -from switchyard.lib.processors.stage_router.dimensions import ( - CodingAgentDimensions, - from_signal, -) from switchyard.lib.processors.stage_router.handoff_notes import ( DEFAULT_ESCALATION_NOTE, HandoffNoteInjector, @@ -27,28 +27,18 @@ pick_capable_first, pick_efficient_first, ) -from switchyard.lib.processors.stage_router.scorer import ( - DEFAULT_WEIGHTS, - ScoreResult, - score, -) __all__ = [ "CONTEXT_KEY", "DEFAULT_ESCALATION_NOTE", - "DEFAULT_WEIGHTS", "CAPABLE", "CAPABLE_TIER", "HandoffNoteInjector", "StageRouterDecisionLog", - "CodingAgentDimensions", "DecisionSource", - "ScoreResult", "TierClassifier", "EFFICIENT", "EFFICIENT_TIER", - "from_signal", "pick_capable_first", "pick_efficient_first", - "score", ] diff --git a/switchyard/lib/processors/stage_router/dimensions.py b/switchyard/lib/processors/stage_router/dimensions.py deleted file mode 100644 index 6c1db768..00000000 --- a/switchyard/lib/processors/stage_router/dimensions.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Scorer-ready view of :class:`ToolResultSignal` — two axes, five signals. - -The stage_router scorer models a coding turn on two axes: - -* **error** — did the recent tool results error? ``severity`` (escalate-only; a - clean-result streak used to sit on this axis but cancelled the windowed error - signal and released escalations too early, so it was removed). -* **production** — is the agent producing code? ``spinning`` / ``exploring`` - (bad poles) vs ``recent_production_intensity`` (good pole). - -``spinning`` and ``exploring`` are mutually exclusive, split by whether the -agent is doing *any* investigative work (reads / plans) in the recent window: -``spinning`` = not even looking, ``exploring`` = looking but not building. - -Every field here is consumed — ``severity`` by the picker override, the rest by -the scorer weights. All counts are over the recent window; see the notes on -:func:`from_signal` for the two windowing caveats. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from switchyard_rust.components import ToolResultSignal - -#: Turn-depth below which stall signals stay quiet — early no-write turns are -#: normal exploration, not a stall. -_STALL_MIN_TURN_DEPTH: int = 8 - - -def _ratio(numerator: int, denominator: int) -> float: - return numerator / denominator if denominator > 0 else 0.0 - - -@dataclass(frozen=True) -class CodingAgentDimensions: - """Normalised, scorer-ready view of a single :class:`ToolResultSignal`.""" - - #: Windowed max error severity in ``[0, 1]`` (used by the picker override; the - #: scorer weights it capped at HARD since CRITICAL short-circuits upstream). - severity: float - #: 1.0 when the recent window has no reads, plans, writes, or edits — the agent - #: is only cycling non-inspecting commands (a struggle signal → strong). - spinning: float - #: 1.0 when the recent window has reads/plans but no writes or edits — the agent - #: is investigating without converging. Full escalation signal → strong; because it - #: persists while the agent isn't producing, it latches sustained escalation on hard tasks. - exploring: float - #: Fraction of recent tool ops that produced code (writes + edits) → weak. - recent_production_intensity: float - #: Largest share of the recent window taken by one identical command-bearing call - #: in ``[0, 1]`` — a weak model looping the same command (churn) → strong. Catches - #: the "busy but not converging" case that severity/spinning miss. - repeated_cmd_ratio: float - - -def from_signal(signal: ToolResultSignal) -> CodingAgentDimensions: - """Project a :class:`ToolResultSignal` onto the two-axis dimension space. - - Windowing notes (both inherent to routing on the normalised request): - - * ``turn_depth`` is a raw *message* count, so its scale varies by wire format - (Anthropic batches tool results into fewer messages than OpenAI-chat). The - ``_STALL_MIN_TURN_DEPTH`` gate is therefore approximate across origins. - * ``severity`` is windowed over the last N tool *results* while the ``recent_*`` - counts are over the last N tool *calls* — usually the same turns, but a - trailing call without a result yet can offset them by one. - """ - recent_ops = ( - signal.recent_write_count - + signal.recent_edit_count - + signal.recent_read_count - + signal.recent_todowrite_count - ) - deep_enough = signal.turn_depth >= _STALL_MIN_TURN_DEPTH - no_production = signal.recent_write_count == 0 and signal.recent_edit_count == 0 - investigating = signal.recent_read_count >= 1 or signal.recent_todowrite_count >= 1 - # spinning vs exploring partition the "not producing" case by investigative - # activity, so at most one fires — no double-counting on the production axis. - spinning = deep_enough and no_production and not investigating - exploring = deep_enough and no_production and investigating - return CodingAgentDimensions( - severity=float(signal.severity), - spinning=1.0 if spinning else 0.0, - exploring=1.0 if exploring else 0.0, - recent_production_intensity=_ratio( - signal.recent_write_count + signal.recent_edit_count, recent_ops - ), - repeated_cmd_ratio=float(signal.repeated_cmd_ratio), - ) - - -__all__ = ["CodingAgentDimensions", "from_signal"] diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index 580377bf..66028fc3 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -1,8 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Two pickers (capable-first / efficient-first) that share override + scorer -logic; differ only in their fallback tier on low-confidence turns.""" +"""Thin Python shell over the Rust stage_router picker. + +The routing logic — the two-axis scorer, the escalate/de-escalate shortcuts, and +tier selection — lives in Rust (``switchyard_components::stage_router::pick_tier``, +exposed as ``stage_pick_tier``). This module only adds the Python-side pieces the +Rust core deliberately leaves to the caller: the ``no_signal`` case, the async LLM +classifier, and decision-source recording.""" import logging from typing import TYPE_CHECKING @@ -12,112 +17,73 @@ DecisionSource, StageRouterDecisionLog, ) -from switchyard.lib.processors.stage_router.dimensions import from_signal -from switchyard.lib.processors.stage_router.scorer import DEFAULT_WEIGHTS, score if TYPE_CHECKING: - from collections.abc import Mapping - from switchyard.lib.processors.stage_router.classifier import TierClassifier from switchyard.lib.proxy_context import ProxyContext - from switchyard_rust.components import ToolResultSignal log = logging.getLogger(__name__) EFFICIENT: int = 0 CAPABLE: int = 1 -# Override thresholds — tunable in one place. Promote to YAML if calibration -# diverges across deployments. -#: Force CAPABLE when the latest tool result hit a CRITICAL severity pattern. -SEVERITY_CRITICAL: float = 1.0 +#: Rust returns the tier as a name; map it back to the picker's int constants. +_TIER: dict[str, int] = {"efficient": EFFICIENT, "capable": CAPABLE} async def pick_capable_first( ctx: "ProxyContext", confidence_threshold: float, classifier: "TierClassifier | None" = None, - weights: "Mapping[str, float]" = DEFAULT_WEIGHTS, decision_log: StageRouterDecisionLog | None = None, ) -> int: """CAPABLE default. EFFICIENT only when the scorer is confidently negative.""" - return await _pick( - ctx, - default_tier=CAPABLE, - confidence_threshold=confidence_threshold, - classifier=classifier, - weights=weights, - decision_log=decision_log, - ) + return await _pick(ctx, "capable_first", confidence_threshold, classifier, decision_log) async def pick_efficient_first( ctx: "ProxyContext", confidence_threshold: float, classifier: "TierClassifier | None" = None, - weights: "Mapping[str, float]" = DEFAULT_WEIGHTS, decision_log: StageRouterDecisionLog | None = None, ) -> int: """EFFICIENT default. CAPABLE only when the scorer is confidently positive.""" - return await _pick( - ctx, - default_tier=EFFICIENT, - confidence_threshold=confidence_threshold, - classifier=classifier, - weights=weights, - decision_log=decision_log, - ) + return await _pick(ctx, "efficient_first", confidence_threshold, classifier, decision_log) async def _pick( ctx: "ProxyContext", - default_tier: int, + picker_mode: str, confidence_threshold: float, classifier: "TierClassifier | None", - weights: "Mapping[str, float]", decision_log: StageRouterDecisionLog | None, ) -> int: - from switchyard_rust.components import ( - get_tool_result_signal, # local import: heavy native module + from switchyard_rust.components import ( # local import: heavy native module + get_tool_result_signal, + stage_pick_tier, ) signal = get_tool_result_signal(ctx) if signal is None: - return _record(ctx, decision_log, "no_signal", default_tier) - - override = _apply_overrides(signal) - if override is not None: - return _record(ctx, decision_log, "override", override) - - # Settled run: a recent test-pass backed by a recent code change (write or edit), - # with no error in the window — safe on the cheap tier for both pickers. The - # severity gate keeps a windowed HARD error from being swallowed as "settled"; - # such a turn falls through to the scorer, where the error routes it to CAPABLE. - if ( - signal.tests_passed - and (signal.recent_write_count + signal.recent_edit_count) >= 1 - and not signal.severity > 0.0 - ): - return _record(ctx, decision_log, "tests_passed", EFFICIENT) - - dimensions = from_signal(signal) - # Both pickers share the scorer: severity / spinning / exploring corroborate - # toward CAPABLE, production / clean-streak toward EFFICIENT. The - # confidence_threshold is the corroboration dial (see scorer docstring); the two - # pickers differ only in the low-confidence default tier below. - result = score(dimensions, weights=weights) - if result.confidence >= confidence_threshold: - tier = CAPABLE if result.score > 0 else EFFICIENT - return _record(ctx, decision_log, "dimensions", tier) + default = EFFICIENT if picker_mode == "efficient_first" else CAPABLE + return _record(ctx, decision_log, "no_signal", default) + + # The Rust core runs the escalate/de-escalate shortcuts and the scorer. + outcome = stage_pick_tier(signal, picker_mode, confidence_threshold) + if outcome.resolved: + return _record(ctx, decision_log, outcome.source, _TIER[outcome.tier]) + # Scorer wasn't confident. Consult the classifier; fall open to the default + # tier when there is none or it can't decide. + default = _TIER[outcome.default_tier] if classifier is None: - return _record(ctx, decision_log, "fall_open", default_tier) + return _record(ctx, decision_log, "fall_open", default) verdict = await classifier.classify(ctx, signal) if verdict == "capable": return _record(ctx, decision_log, "llm-classifier", CAPABLE) if verdict == "efficient": return _record(ctx, decision_log, "llm-classifier", EFFICIENT) - return _record(ctx, decision_log, "fall_open", default_tier) + return _record(ctx, decision_log, "fall_open", default) def _record( @@ -137,22 +103,4 @@ def _record( return tier -def _apply_overrides(signal: "ToolResultSignal") -> int | None: - """Non-negotiable, signal-derived shortcuts that bypass the scorer. - - A CRITICAL severity or a context compaction always forces CAPABLE; both outrank - the settled-run (`tests_passed`) shortcut handled in :func:`_pick`. - - Compaction resets the router's accumulated signals, so a task that had escalated - de-escalates straight back to weak. Once the context has been compacted we force — - and, because the summary stays in the prefix on every subsequent turn, hold — the - strong tier, which is where a task hard enough to overflow its context belongs. - """ - if signal.compacted: - return CAPABLE - if signal.severity >= SEVERITY_CRITICAL: - return CAPABLE - return None - - __all__ = ["CAPABLE", "EFFICIENT", "pick_capable_first", "pick_efficient_first"] diff --git a/switchyard/lib/processors/stage_router/scorer.py b/switchyard/lib/processors/stage_router/scorer.py deleted file mode 100644 index 13281ec8..00000000 --- a/switchyard/lib/processors/stage_router/scorer.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Signed linear scorer over two axes (error, production), tanh-squashed to ``(-1, +1)``. - -Each signal contributes a fixed weight; WRONG signals push toward CAPABLE (+), -PROGRESS signals toward EFFICIENT (−). The raw weighted sum is squashed with -``tanh(gain * raw)`` and ``confidence = abs(score)``. - -**On the threshold (read this before tuning):** the raw sum is small — one maxed -signal is ``±0.10``, two corroborating signals ``±0.20``. The gain spreads that -into a usable confidence range; it does **not** make the threshold an integer -"signals-to-clear" count. Empirically the dial reads: - - conf 0.462 one full signal (a HARD error, spinning, or exploring) - conf 0.762 two full signals (e.g. severity + exploring, severity + spinning) - -So a threshold near 0.3 escalates on ~one signal, ~0.5 needs ~1.5, ~0.7 needs two -to corroborate. The reachable range is ``(0, ~0.76)`` for these two axes — a -threshold above ~0.76 can't be met, so it always defers to the classifier/default. -""" - -from __future__ import annotations - -import math -from collections.abc import Mapping -from dataclasses import dataclass, field - -from switchyard.lib.processors.stage_router.dimensions import CodingAgentDimensions - -#: Gain applied before the tanh squash — spreads the small raw sum across the -#: usable confidence range (see the module docstring for the resulting dial). Load- -#: bearing: without it, confidence would cap near ±0.20 and mid/high thresholds -#: would be unreachable. -_SCORE_GAIN: float = 5.0 - -#: Max severity the scorer ever sees: ``1.0`` (critical) is caught by the picker -#: override, so ``0.7`` (hard) is the strongest error that reaches the linear sum. -#: Used to normalise ``severity`` so a HARD error contributes exactly ``_SIGNAL_UNIT``. -_HARD_SEVERITY: float = 0.7 -#: Weight one maxed signal contributes. Small enough that no single axis pegs the -#: decision; corroboration across the two axes is what pushes confidence up. -_SIGNAL_UNIT: float = 0.10 - -#: "Something is wrong" → CAPABLE (strong). ``exploring`` (reading/planning without -#: producing) is a FULL escalation signal: it clears the threshold on its own, and -#: because the condition persists while the agent isn't producing, it holds strong -#: across the whole stretch — the "latch" that sustains escalation on hard debugging -#: tasks (verified against runs where a struggling agent reads-without-writing for -#: dozens of turns; treating it as neutral routed those to the weak tier and lost them). -#: -#: ``repeated_cmd_ratio`` was dropped here: an ablation over 178 task-runs (two -#: Nemotron deployments) found 0/71 of its escalations were real command loops — its -#: ``max_count / window`` normalisation makes a single Bash call read as ratio 0.20, -#: so it fired a low-grade CAPABLE bias on any turn that touched the shell, adding -#: ~8% Opus with no accuracy payoff. The Rust signal still computes it for -#: observability; it just no longer influences routing. Re-add with a ``max_count >= 3`` -#: gate if a genuine same-command-loop detector is wanted. -_WRONG_SIGNALS: tuple[str, ...] = ("severity", "spinning", "exploring") -#: "Making progress" → EFFICIENT (weak). De-escalation is driven by real production -#: (writes/edits), not by a clean-result streak: a streak of clean results actively -#: cancelled the (windowed) error signal on the same axis and released escalations too -#: early — trace-verified to suppress escalation entirely on some tasks — so the error -#: axis is now escalate-only (severity) and progress is production. -_PROGRESS_SIGNALS: tuple[str, ...] = ("recent_production_intensity",) -#: Max value each signal reaches, used to normalise so a maxed signal contributes -#: ``_SIGNAL_UNIT``. Defaults to 1.0 for ``[0, 1]`` gates/ratios. -_MAX_VALUE: Mapping[str, float] = {"severity": _HARD_SEVERITY} - - -def _build_weights(unit: float = _SIGNAL_UNIT) -> dict[str, float]: - """Signed, fixed linear weights: wrong → CAPABLE (+), progress → EFFICIENT (−). - - Every maxed signal contributes ``unit`` (``severity`` is normalised by its HARD - cap so it too lands at ``unit``). - """ - weights: dict[str, float] = {} - for name in _WRONG_SIGNALS: - weights[name] = unit / _MAX_VALUE.get(name, 1.0) - for name in _PROGRESS_SIGNALS: - weights[name] = -unit / _MAX_VALUE.get(name, 1.0) - return weights - - -#: Fixed scorer weights. Corroboration across axes is dialed by confidence_threshold. -DEFAULT_WEIGHTS: Mapping[str, float] = _build_weights() - - -@dataclass(frozen=True) -class ScoreResult: - """Output of :func:`score`. ``confidence == abs(score)`` by construction.""" - - score: float - confidence: float - contributions: Mapping[str, float] = field(default_factory=dict) - - -def score( - dimensions: CodingAgentDimensions, - *, - weights: Mapping[str, float] = DEFAULT_WEIGHTS, -) -> ScoreResult: - """Score ``dimensions``; raw weighted sum is tanh-squashed into ``(-1, +1)``. - - ``contributions`` are the raw per-signal weighted values (pre-squash, so they - sum to the raw score); ``score`` is ``tanh(gain * raw)``; ``confidence`` its - magnitude. Positive ``score`` → CAPABLE, negative → EFFICIENT. - """ - contributions: dict[str, float] = {} - raw = 0.0 - for field_name, weight in weights.items(): - value = getattr(dimensions, field_name, 0.0) - contribution = value * weight - contributions[field_name] = contribution - raw += contribution - squashed = math.tanh(_SCORE_GAIN * raw) - return ScoreResult(score=squashed, confidence=abs(squashed), contributions=contributions) - - -__all__ = ["DEFAULT_WEIGHTS", "ScoreResult", "score"] diff --git a/switchyard_rust/components.py b/switchyard_rust/components.py index 74f86864..f7578950 100644 --- a/switchyard_rust/components.py +++ b/switchyard_rust/components.py @@ -13,9 +13,7 @@ { "AnthropicNativeBackend", "BackendFormat", - "ContextSignals", "DimensionCollector", - "DimensionScore", "EndpointConfig", "IntakeQueueFullPolicy", "IntakeRequestMetadata", @@ -27,31 +25,30 @@ "MultiLlmBackend", "OpenAiNativeBackend", "OpenAiPassthroughBackend", + "PickOutcome", "RandomRoutingProcessorConfig", "RequestMetadata", "ResponseFlag", "ResponseSignalCollector", "ResponseSignals", - "ScoringConfig", "StatsAccumulator", "StatsLlmBackend", "StatsRequestProcessor", "StatsResponseProcessor", "ToolResultSignal", "extract_response_signals", - "get_context_signals", "get_response_signals", "get_tool_result_signal", "set_stats_route_label", + "stage_pick_tier", + "stage_score_signal", } ) if TYPE_CHECKING: AnthropicNativeBackend: type[Any] BackendFormat: type[Any] - ContextSignals: type[Any] DimensionCollector: type[Any] - DimensionScore: type[Any] EndpointConfig: type[Any] IntakeQueueFullPolicy: type[Any] IntakeRequestMetadata: type[Any] @@ -63,22 +60,23 @@ MultiLlmBackend: type[Any] OpenAiNativeBackend: type[Any] OpenAiPassthroughBackend: type[Any] + PickOutcome: type[Any] RandomRoutingProcessorConfig: type[Any] RequestMetadata: type[Any] ResponseFlag: type[Any] ResponseSignalCollector: type[Any] ResponseSignals: type[Any] - ScoringConfig: type[Any] StatsAccumulator: type[Any] StatsLlmBackend: type[Any] StatsRequestProcessor: type[Any] StatsResponseProcessor: type[Any] ToolResultSignal: type[Any] extract_response_signals: Any - get_context_signals: Any get_response_signals: Any get_tool_result_signal: Any set_stats_route_label: Any + stage_pick_tier: Any + stage_score_signal: Any def __getattr__(name: str) -> object: diff --git a/tests/test_dimension_collector.py b/tests/test_dimension_collector.py deleted file mode 100644 index 0e10b506..00000000 --- a/tests/test_dimension_collector.py +++ /dev/null @@ -1,177 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end Python tests for the Rust-backed ``DimensionCollector``. - -Mirrors the coverage of the deleted -``switchyard/experimental/rules_routing/tests/test_dimension_collector.py`` -against the new Rust port exposed through ``switchyard_rust.components``. -""" - -from __future__ import annotations - -from typing import Any, cast - -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.components import ( - ContextSignals, - DimensionCollector, - DimensionScore, - ScoringConfig, - get_context_signals, -) -from switchyard_rust.core import ChatRequest - - -def _request(prompt: str) -> ChatRequest: - return ChatRequest.openai_chat( - cast( - Any, - { - "model": "client-model", - "messages": [{"role": "user", "content": prompt}], - }, - ), - ) - - -def _default_config() -> ScoringConfig: - return ScoringConfig( - code_keywords=["def", "class", "function"], - reasoning_keywords=["prove", "theorem", "derive"], - simple_keywords=["hello", "what is", "define"], - technical_keywords=["kubernetes", "distributed", "algorithm", "protocol"], - creative_keywords=["story", "poem", "brainstorm"], - imperative_verbs=["build", "create", "implement"], - constraint_indicators=["at most", "within", "no more than"], - output_format_keywords=["json", "yaml", "csv"], - reference_keywords=["the code above", "the api docs"], - negation_keywords=["not", "never", "except", "unless"], - domain_specific_keywords=["quantum", "fpga", "genomics"], - ) - - -async def test_dimension_collector_stamps_context_signals_into_proxy_context() -> None: - """Hot-path assertion: ``get_context_signals`` returns the stamped record.""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("def hello(): class Foo: pass")) - - signals = get_context_signals(ctx) - assert isinstance(signals, ContextSignals) - assert len(signals.dimensions) == 14 - code_presence = next(d for d in signals.dimensions if d.name == "codePresence") - assert isinstance(code_presence, DimensionScore) - assert code_presence.score == 1.0 - - -async def test_dimensions_are_emitted_in_canonical_order() -> None: - """Order is a public contract for downstream estimators (e.g. weighted sums).""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("Hello world.")) - - signals = get_context_signals(ctx) - assert signals is not None - assert [d.name for d in signals.dimensions] == [ - "tokenCount", - "codePresence", - "reasoningMarkers", - "technicalTerms", - "creativeMarkers", - "simpleIndicators", - "imperativeVerbs", - "constraintCount", - "outputFormat", - "referenceComplexity", - "negationComplexity", - "domainSpecificity", - "multiStepPatterns", - "questionComplexity", - ] - - -async def test_token_count_short_pushes_negative_score() -> None: - """A bare ``hello`` should score short (chars/4 = 1 ≪ default short=50).""" - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("hello")) - - signals = get_context_signals(ctx) - assert signals is not None - token_dim = next(d for d in signals.dimensions if d.name == "tokenCount") - assert token_dim.score == -1.0 - - -async def test_simple_indicators_pull_negative_for_chitchat() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("hello, what is the meaning of life?")) - - signals = get_context_signals(ctx) - assert signals is not None - simple_dim = next(d for d in signals.dimensions if d.name == "simpleIndicators") - assert simple_dim.score == -1.0 - - -async def test_multi_step_patterns_detect_first_then_chain() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("first do x then do y")) - - signals = get_context_signals(ctx) - assert signals is not None - multi = next(d for d in signals.dimensions if d.name == "multiStepPatterns") - assert multi.score == 0.5 - - -async def test_question_complexity_fires_above_three_questions() -> None: - collector = DimensionCollector(_default_config()) - ctx = ProxyContext() - - await collector.process(ctx, _request("A? B? C? D? E?")) - - signals = get_context_signals(ctx) - assert signals is not None - questions = next(d for d in signals.dimensions if d.name == "questionComplexity") - assert questions.score == 0.5 - - -async def test_collector_without_args_uses_populated_rust_defaults() -> None: - """No-arg `DimensionCollector()` picks up `ScoringConfig::default()`. - - The Rust-side `Default` ships populated keyword lists. A bland prompt like ``"just some text"`` should still - emit the full 14-dimension vector with sensible zeros for - non-matching scorers. - """ - collector = DimensionCollector() - ctx = ProxyContext() - - await collector.process(ctx, _request("just some text")) - - signals = get_context_signals(ctx) - assert signals is not None - assert len(signals.dimensions) == 14 - # `"just some text"` is short (< 50 token estimate) so `tokenCount` - # fires negative; no keyword scorer should hit on this prompt. - by_name = {d.name: d.score for d in signals.dimensions} - assert by_name["tokenCount"] == -1.0 - keyword_dims = { - "codePresence", "reasoningMarkers", "technicalTerms", - "creativeMarkers", "simpleIndicators", "imperativeVerbs", - "constraintCount", "outputFormat", "referenceComplexity", - "negationComplexity", "domainSpecificity", - } - for name in keyword_dims: - assert by_name[name] == 0.0, f"{name} unexpectedly fired" - - -async def test_get_context_signals_returns_none_when_not_stamped() -> None: - """Reader must not fabricate a record when no collector has run.""" - ctx = ProxyContext() - assert get_context_signals(ctx) is None diff --git a/tests/test_stage_router_scorer.py b/tests/test_stage_router_scorer.py deleted file mode 100644 index 3b721145..00000000 --- a/tests/test_stage_router_scorer.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the stage-router dimensions + two-axis scorer.""" - -from __future__ import annotations - -import math - -import pytest - -from switchyard.lib.processors.stage_router.dimensions import ( - CodingAgentDimensions, - from_signal, -) -from switchyard.lib.processors.stage_router.scorer import ( - _SCORE_GAIN, - DEFAULT_WEIGHTS, - score, -) -from switchyard_rust.components import DimensionCollector -from switchyard_rust.core import ChatRequest, ProxyContext - - -def _zero() -> CodingAgentDimensions: - return CodingAgentDimensions( - severity=0.0, - spinning=0.0, - exploring=0.0, - recent_production_intensity=0.0, - repeated_cmd_ratio=0.0, - ) - - -def _with(**kw: float) -> CodingAgentDimensions: - return CodingAgentDimensions(**{**_zero().__dict__, **kw}) - - -def test_zero_signal_scores_to_zero(): - result = score(_zero()) - assert result.score == 0.0 - assert result.confidence == 0.0 - - -def test_wrong_signals_point_capable(): - assert score(_with(severity=0.7)).score > 0 - assert score(_with(spinning=1.0)).score > 0 - assert score(_with(exploring=1.0)).score > 0 - - -def test_progress_signal_points_efficient(): - assert score(_with(recent_production_intensity=1.0)).score < 0 - - -def test_hard_severity_and_spinning_contribute_one_unit(): - # severity is normalised by its HARD cap (0.7), so a HARD error lands at the - # same unit as a maxed boolean signal like spinning. - hard = score(_with(severity=0.7)) - spin = score(_with(spinning=1.0)) - assert hard.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) - assert spin.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) - - -def test_exploring_is_a_full_escalation_signal(): - """exploring is a full-weight WRONG signal → clears a default threshold on its own - (it's the persistent 'reading-without-producing' latch), same weight as spinning.""" - explore = score(_with(exploring=1.0)) - spin = score(_with(spinning=1.0)) - assert explore.score > 0 - assert explore.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.10)) # ~0.462 - assert explore.confidence == pytest.approx(spin.confidence) - assert explore.confidence > 0.30 # escalates alone at a typical threshold - - -def test_repeated_cmd_ratio_does_not_influence_routing(): - """repeated_cmd_ratio was dropped from the WRONG signals (ablation: 0/71 of its - escalations were real loops — it fired on any single shell command). It is still - carried on the dimensions for observability but must contribute nothing to score.""" - churn = score(_with(repeated_cmd_ratio=1.0)) - assert churn.score == 0.0 - assert churn.confidence == 0.0 - assert "repeated_cmd_ratio" not in churn.contributions - - -def test_corroboration_raises_confidence(): - """Two agreeing wrong signals corroborate to higher confidence than one.""" - one = score(_with(severity=0.7)) - two = score(_with(severity=0.7, spinning=1.0)) - assert two.confidence > one.confidence - assert two.score == pytest.approx(math.tanh(_SCORE_GAIN * 0.20)) # ~0.762 - - -def test_axes_can_cancel_to_neutral(): - """A turn that both errored and produced nets toward zero confidence.""" - dims = _with(severity=0.7, recent_production_intensity=1.0) - result = score(dims) - assert result.confidence < 0.30 # roughly cancels → defers to classifier/default - - -def test_tanh_saturates_smoothly_via_weight_override(): - dims = _with(severity=1.0) - high = score(dims, weights={"severity": 5.0}) # raw 5.0 → tanh(25) ≈ 1.0 - assert high.score == pytest.approx(1.0) - low = score(dims, weights={"severity": -5.0}) - assert low.score == pytest.approx(-1.0) - - -def test_custom_weights_can_invert_decision(): - dims = _with(severity=1.0) - assert score(dims, weights=DEFAULT_WEIGHTS).score > 0 - assert score(dims, weights={"severity": -0.5}).score < 0 - - -def test_contributions_are_raw_presquash(): - dims = _with(recent_production_intensity=0.5) - result = score(dims) - raw = sum(result.contributions.values()) - assert result.score == pytest.approx(math.tanh(_SCORE_GAIN * raw)) - - -@pytest.mark.asyncio -async def test_from_signal_normalises_real_extracted_signal(): - """End-to-end: DimensionCollector → ToolResultSignal → CodingAgentDimensions.""" - collector = DimensionCollector() - ctx = ProxyContext() - request = ChatRequest.openai_chat({ - "messages": [ - {"role": "assistant", - "tool_calls": [{"function": {"name": "Write", "arguments": "{}"}}]}, - {"role": "tool", "tool_call_id": "x", "content": "ok"}, - {"role": "user", "content": "next"}, - ] - }) - await collector.process(ctx, request) - from switchyard_rust.components import get_tool_result_signal - signal = get_tool_result_signal(ctx) - assert signal is not None - dims = from_signal(signal) - assert 0.0 <= dims.severity <= 1.0 - assert dims.recent_production_intensity > 0 # we issued one Write call - # too shallow (turn_depth < 8) for a stall signal to fire - assert dims.spinning == 0.0 - assert dims.exploring == 0.0 From a67b94be78f0f7c2242807cbc1ac751be80f48e7 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 23 Jul 2026 16:58:26 -0700 Subject: [PATCH 16/21] chore(benchmark): remove stray stage-router routing-profile configs from the PR Signed-off-by: Sabhatina Selvam --- ...outer-evalsnem-opus-ef-t30-w5-handoff.yaml | 37 ------------------- ...ge-router-nemu-opus-ef-t50-w5-handoff.yaml | 33 ----------------- 2 files changed, 70 deletions(-) delete mode 100644 benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml delete mode 100644 benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml diff --git a/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml b/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml deleted file mode 100644 index c7f8b06f..00000000 --- a/benchmark/routing-profiles/tb2-1-stage-router-evalsnem-opus-ef-t30-w5-handoff.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# A/B variant of tb2-1-stage-router-evalsnem-opus-ef-t30-w5 with tier-transition -# handoff notes ENABLED (escalation direction only; de-escalation left off). -# Everything else — ef · evals-nemotron weak · Opus 4.8 @ high · window 5 · t0.30 — -# is identical, so a diff vs the base run isolates the note's effect. -routes: - switchyard: - type: stage_router - picker: efficient_first - confidence_threshold: 0.30 - signal_recent_window: 5 - fallback_target_on_evict: strong - enable_stats: true - handoff_notes: - enabled: true - only_on_wrong_signal_escalation: true - # escalation_note uses the built-in default; override here to customise. - # deescalation_note: left unset (strong→weak hand-backs inject nothing). - strong: - model: azure/anthropic/claude-opus-4-8 - api_key: ${OPENAI_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - format: anthropic - timeout_secs: 600 - extra_body: - output_config: - effort: high - weak: - model: nvidia/nvidia/evals-nemotron-ultra-3 - api_key: ${OPENAI_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - format: openai - timeout_secs: 600 - extra_headers: - X-Inference-Priority: batch - extra_body: - chat_template_kwargs: - enable_thinking: true diff --git a/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml b/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml deleted file mode 100644 index 79b46e1f..00000000 --- a/benchmark/routing-profiles/tb2-1-stage-router-nemu-opus-ef-t50-w5-handoff.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Stage-router ef · nemotron-3-ultra weak · Opus 4.8 @ high · window 5 · t0.50 · handoff notes ON. -# Higher threshold vs tb2-1-stage-router-nemu-opus-ef-t30-w5-handoff — tighter escalation gate. -routes: - switchyard: - type: stage_router - picker: efficient_first - confidence_threshold: 0.50 - signal_recent_window: 5 - fallback_target_on_evict: strong - enable_stats: true - handoff_notes: - enabled: true - only_on_wrong_signal_escalation: true - strong_system_prompt: "Be concise. Prefer short, targeted tool calls over lengthy explanations. Do not re-examine what is already working — focus only on the immediate blocker. Avoid restating context the conversation already contains." - weak_system_prompt: "Be concise and efficient. One focused tool call per turn. If the same approach has failed twice, stop and clearly report what failed rather than retrying. Do not explain what you are about to do — just do it." - strong: - model: azure/anthropic/claude-opus-4-8 - api_key: ${OPENAI_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - format: anthropic - timeout_secs: 600 - extra_body: - output_config: - effort: high - weak: - model: nvidia/nvidia/nemotron-3-ultra - api_key: ${OPENAI_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - format: openai - timeout_secs: 600 - extra_body: - chat_template_kwargs: - enable_thinking: false From 5fbe40b187b0f65911365edbe00a4bc726532e6f Mon Sep 17 00:00:00 2001 From: Greg Clark Date: Thu, 23 Jul 2026 20:44:57 -0400 Subject: [PATCH 17/21] chore: transfer changes from tool_signals to libsy Signed-off-by: Greg Clark --- crates/libsy/src/signal/tool_signals.rs | 340 +++-- .../src/dimension_collector/tool_signals.rs | 1237 +---------------- 2 files changed, 222 insertions(+), 1355 deletions(-) diff --git a/crates/libsy/src/signal/tool_signals.rs b/crates/libsy/src/signal/tool_signals.rs index a7020fa2..91b4ffaa 100644 --- a/crates/libsy/src/signal/tool_signals.rs +++ b/crates/libsy/src/signal/tool_signals.rs @@ -65,7 +65,14 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ ( "no_such_file", HARD, - &["filenotfounderror:", "no such file or directory"], + &[ + "filenotfounderror:", + "no such file or directory", + // Claude Code Read-tool miss. Anchored as "file does not exist" (not a + // bare "does not exist", which fires on `ls` output and prose) — trace- + // mined across 1006 local trajectories at 22 true / 2 false positives. + "file does not exist", + ], ), // SOFT: plain non-zero exit without a recognisable exception traceback. ( @@ -132,8 +139,7 @@ static BASH_READ_PATTERNS: &[&str] = &[ static READ_TOOL_NAMES: &[&str] = &["read", "view"]; -// Planning / scratchpad tool calls. Used by Opus as a struggle indicator -// in the capable-first picker direction. +// Planning / scratchpad tool calls — investigative (non-producing) activity. // `update_plan` is codex's equivalent of `todowrite`. static PLAN_TOOL_NAMES: &[&str] = &["todowrite", "todo_write", "todo", "update_plan"]; @@ -168,13 +174,13 @@ static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "er // "0 errors" summaries on a clean run are not misread as failures. static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"]; -/// Default sliding-window size for `recent_*` counts. +/// Default sliding-window size for `recent_*` counts and windowed severity. /// -/// Calibrated against agent trajectories where a 3-call horizon is enough -/// to capture the "what is the agent doing right now" signal without -/// over-smoothing short tasks. Override per-extractor by calling -/// [`extract_tool_signals_with_window`] directly. -pub const DEFAULT_RECENT_WINDOW: usize = 3; +/// A 5-call horizon captures "what is the agent doing right now" while keeping +/// signals sticky — an error or stall persists a few recovery turns instead of +/// flickering off the moment one clean result lands. Override per-extractor by +/// calling [`extract_tool_signals_with_window`] directly. +pub const DEFAULT_RECENT_WINDOW: usize = 5; // ─── output type ───────────────────────────────────────────────────────────── @@ -182,18 +188,19 @@ pub const DEFAULT_RECENT_WINDOW: usize = 3; /// via [`crate::get_tool_result_signal`]. #[derive(Clone, Debug, Default)] pub struct ToolSignals { + /// Max severity across the recent window (last `recent_window` tool results): /// `0.0` clean · `0.3` soft (exit_nonzero) · `0.7` hard · `1.0` critical. + /// Windowed so an error persists through the recovery turns instead of clearing + /// the instant the next result is clean. pub severity: f32, - /// Error pattern names that fired in the most recent tool result. - pub patterns: Vec, /// Consecutive clean tool results back from the most recent. `0` if the last failed. pub no_error_streak: u32, pub edit_count: u32, pub write_count: u32, /// Read-type calls (Read tool + read-like Bash). Used by the build-pit gate. pub read_count: u32, - /// TodoWrite tool calls. Strong fail predictor for Opus; used by the - /// `capable_first` drop-to-weak gate. + /// TodoWrite / planning tool calls. Investigative (non-producing) activity — + /// recent todowrites distinguish `exploring` from `spinning` in the scorer. pub todowrite_count: u32, /// Edit-type calls within the last [`RECENT_WINDOW`] tool calls. pub recent_edit_count: u32, @@ -203,15 +210,21 @@ pub struct ToolSignals { pub recent_read_count: u32, /// TodoWrite calls within the last [`RECENT_WINDOW`] tool calls. pub recent_todowrite_count: u32, - /// Consecutive trailing tool calls that hit no Write/Edit/Read/Plan - /// patterns — proxy for the "stuck in non-Read Bash" build-pit loop. + /// Consecutive trailing tool calls in the `Other` category (no Write/Edit/Read/ + /// Plan match). Surfaced in the classifier state summary; not scored directly. pub pure_bash_streak: u32, /// At least one of the last three tool results matched a test-pass pattern. pub tests_passed: bool, - /// Message-count proxy for turn depth. + /// Message-count proxy for turn depth. Wire-format dependent (Anthropic batches + /// tool results into fewer messages than OpenAI-chat), so gates keyed on it are + /// approximate across request origins. pub turn_depth: u32, - /// Char count of the last user message. - pub prompt_char_count: u32, + /// The request carries a context-compaction summary (the agent's context was + /// summarised after overflowing). Compaction resets the router's accumulated + /// signals, so a task that was on the strong tier de-escalates back to weak — the + /// picker uses this to force + hold the strong tier. Self-latching: the summary + /// stays in the context prefix on every subsequent turn. + pub compacted: bool, } impl ToolSignals { @@ -269,7 +282,7 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { // ─── extraction entry point ─────────────────────────────────────────────────── -/// Extract all tool-execution signals from a [`ChatRequest`]. +/// Extract all tool-execution signals from a [`Request`]. /// /// Dispatches on the request's wire format: /// @@ -280,7 +293,7 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { /// * **OpenAI responses** — `input[]` with `type: "function_call_output"`; /// `type: "function_call"` for call names. /// -/// Returns [`ToolSignals::default()`] when the body is absent or +/// Returns [`ToolSignals::default()`] when the wire format or body is absent or /// the messages list is empty — callers can always read `signal.severity`. fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals { let Some(Some(wire_format)) = request.metadata.as_ref().map(|m| m.wire_format) else { @@ -293,14 +306,17 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> return ToolSignals::default(); }; - match wire_format { + let (mut signal, entries) = match wire_format { WireFormat::OpenAiChat => { let messages = obj .get("messages") .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_openai_chat(messages, recent_window) + ( + extract_from_messages_openai_chat(messages, recent_window), + messages, + ) } WireFormat::AnthropicMessages => { let messages = obj @@ -308,7 +324,10 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_messages_anthropic(messages, recent_window) + ( + extract_from_messages_anthropic(messages, recent_window), + messages, + ) } WireFormat::OpenAiResponses => { let items = obj @@ -316,17 +335,43 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> .and_then(Value::as_array) .map(Vec::as_slice) .unwrap_or(&[]); - extract_from_input_responses(items, recent_window) + (extract_from_input_responses(items, recent_window), items) } - } + }; + + // Compaction is detected anywhere in the message/item contents (the summary stays + // in the prefix on every subsequent turn, so this self-latches once it fires). + signal.compacted = entries.iter().any(|m| { + m.as_object() + .is_some_and(|o| content_has_compaction_marker(o.get("content"))) + }); + signal } // ─── format-specific extractors ────────────────────────────────────────────── +/// Distinctive preamble Claude Code injects as a user message when it compacts an +/// overflowed context. Matched case-insensitively; normal task text never contains it. +const COMPACTION_MARKER: &str = "session is being continued"; + +/// True when a user-message content (string or text blocks) carries the compaction +/// summary preamble. +fn content_has_compaction_marker(content: Option<&Value>) -> bool { + match content { + Some(Value::String(s)) => s.to_lowercase().contains(COMPACTION_MARKER), + Some(Value::Array(blocks)) => blocks.iter().any(|b| { + b.as_object() + .and_then(|o| o.get("text")) + .and_then(Value::as_str) + .is_some_and(|t| t.to_lowercase().contains(COMPACTION_MARKER)) + }), + _ => false, + } +} + fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -367,26 +412,16 @@ fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) - } } } - "user" => { - prompt_char_count = user_message_char_count(obj.get("content")); - } _ => {} } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for msg in messages { let Some(obj) = msg.as_object() else { continue }; @@ -396,31 +431,14 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> match role { "user" => { if let Some(Value::Array(blocks)) = content { - let mut saw_text = false; for block in blocks { let Some(b) = block.as_object() else { continue }; - match b.get("type").and_then(Value::as_str) { - Some("tool_result") => { - if let Some(text) = content_to_text(b.get("content")) { - tool_texts.push(text); - } + if b.get("type").and_then(Value::as_str) == Some("tool_result") { + if let Some(text) = content_to_text(b.get("content")) { + tool_texts.push(text); } - Some("text") => { - if let Some(t) = b.get("text").and_then(Value::as_str) { - prompt_char_count = t.len() as u32; - saw_text = true; - } - } - _ => {} - } - } - if !saw_text { - if let Some(text_val) = content { - prompt_char_count = user_message_char_count(Some(text_val)); } } - } else { - prompt_char_count = user_message_char_count(content); } } "assistant" => { @@ -450,26 +468,18 @@ fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> } } - build_signal( - tool_texts, - tool_calls, - messages.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) } fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolSignals { let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - let mut prompt_char_count: u32 = 0; for item in items { let Some(obj) = item.as_object() else { continue; }; let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); match item_type { "function_call_output" => { @@ -495,21 +505,11 @@ fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolSi command, }); } - _ => { - if role == "user" { - prompt_char_count = user_message_char_count(obj.get("content")); - } - } + _ => {} } } - build_signal( - tool_texts, - tool_calls, - items.len() as u32, - prompt_char_count, - recent_window, - ) + build_signal(tool_texts, tool_calls, items.len() as u32, recent_window) } // ─── aggregation ───────────────────────────────────────────────────────────── @@ -518,14 +518,21 @@ fn build_signal( tool_texts: Vec, tool_calls: Vec, turn_depth: u32, - prompt_char_count: u32, recent_window: usize, ) -> ToolSignals { - let (severity, patterns) = if let Some(last) = tool_texts.last() { - classify_text(last) - } else { - (0.0, Vec::new()) - }; + // Windowed severity: take the MAX severity across the last `recent_window` tool + // results rather than only the last one. An error's severity then persists for + // the recent window and decays out of it — parallel to the windowed `recent_*` + // counts — so a fix written a couple of turns after an error still routes on the + // error signal instead of the router flapping straight back to the weak tier. + let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); + let mut severity = 0.0f32; + for text in &tool_texts[sev_start..] { + let (sev, _patterns) = classify_text(text); + if sev > severity { + severity = sev; + } + } let no_error_streak = compute_no_error_streak(&tool_texts); @@ -581,11 +588,10 @@ fn build_signal( } } - let tests_passed = detect_tests_passed(&tool_texts); + let tests_passed = detect_tests_passed(&tool_texts, recent_window); ToolSignals { severity, - patterns, no_error_streak, edit_count, write_count, @@ -598,7 +604,9 @@ fn build_signal( pure_bash_streak, tests_passed, turn_depth, - prompt_char_count, + // Set by extract_tool_signals_with_window after the format-specific extract, + // which scans all message contents for the compaction marker. + compacted: false, } } @@ -628,24 +636,6 @@ fn content_to_text(content: Option<&Value>) -> Option { } } -/// Character count of a user-message content value. -fn user_message_char_count(content: Option<&Value>) -> u32 { - match content { - Some(Value::String(s)) => s.len() as u32, - Some(Value::Array(blocks)) => blocks - .iter() - .filter_map(|b| { - b.as_object() - .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - }) - .map(|s| s.len() as u32) - .sum(), - _ => 0, - } -} - /// Match `text` against the error pattern table. /// /// Returns `(max_severity, matched_pattern_names)`. @@ -674,13 +664,9 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 { streak } -fn detect_tests_passed(tool_texts: &[String]) -> bool { - let recent = if tool_texts.len() > 3 { - &tool_texts[tool_texts.len() - 3..] - } else { - tool_texts - }; - recent.iter().any(|text| { +fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool { + let start = tool_texts.len().saturating_sub(recent_window.max(1)); + tool_texts[start..].iter().any(|text| { let lower = text.to_lowercase(); TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) @@ -782,6 +768,23 @@ mod tests { assert_eq!(sev, HARD); } + #[test] + fn file_does_not_exist_is_hard() { + // Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives). + let (sev, patterns) = + classify_text("Error: File does not exist. Note: current working directory is /app."); + assert_eq!(sev, HARD); + assert!(patterns.contains(&"no_such_file".to_string())); + } + + #[test] + fn bare_does_not_exist_stays_clean() { + // Precision guard: only the anchored "file does not exist" fires, so a bare + // "does not exist" in prose or directory output must not trip a false error. + let (sev, _) = classify_text("The directory does not exist yet, creating it now."); + assert_eq!(sev, 0.0); + } + #[test] fn no_error_streak_all_clean() { let texts = vec!["ok".to_string(), "all good".to_string()]; @@ -800,38 +803,48 @@ mod tests { #[test] fn tests_passed_detects_pytest_output() { - assert!(detect_tests_passed(&[ - "====== 5 passed in 0.12s ======".to_string() - ])); + assert!(detect_tests_passed( + &["====== 5 passed in 0.12s ======".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_ignores_partial_failures() { - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ])); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); + } + + #[test] + fn severity_is_windowed_over_recent_results() { + // An error two results back, then two clean results. + let request = openai_chat_request(json!({ + "messages": [ + {"role": "tool", "tool_call_id": "1", + "content": "Traceback (most recent call last):\n ValueError"}, + {"role": "tool", "tool_call_id": "2", "content": "ok"}, + {"role": "tool", "tool_call_id": "3", "content": "ok"}, + ] + })); + // window covers the error → severity persists (max over the window) + assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD); + // window of 1 sees only the last (clean) result → severity has decayed out + assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0); } #[test] fn extract_openai_chat_tool_results() { - let raw_request = Some(json!({ + let request = openai_chat_request(json!({ "messages": [ {"role": "user", "content": "do something"}, {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, {"role": "tool", "tool_call_id": "1", "content": "Traceback (most recent call last):\n ValueError"}, ] })); - let request = Request { - raw_request, - metadata: Some(Metadata { - wire_format: Some(WireFormat::OpenAiChat), - ..Default::default() - }), - ..Default::default() - }; let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, HARD); - assert!(sig.patterns.contains(&"traceback".to_string())); assert_eq!(sig.edit_count, 1); assert_eq!(sig.turn_depth, 3); } @@ -865,9 +878,9 @@ mod tests { } #[test] - fn recent_window_counts_only_last_three_tool_calls() { - // 5 writes + 1 edit at the end → recent window of 3 should see - // 1 edit + 2 writes (not all 5 writes). + fn recent_window_counts_only_last_default_window_tool_calls() { + // 5 writes + 1 edit at the end → the default window (5) should see + // the last 5 calls: 1 edit + 4 writes (not all 6 calls). let request = openai_chat_request(json!({ "messages": [ {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, @@ -887,7 +900,7 @@ mod tests { let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 5); assert_eq!(sig.edit_count, 1); - assert_eq!(sig.recent_write_count, 2); + assert_eq!(sig.recent_write_count, 4); assert_eq!(sig.recent_edit_count, 1); } @@ -921,6 +934,29 @@ mod tests { assert_eq!(wide.recent_edit_count, 1); } + #[test] + fn compaction_marker_sets_compacted() { + // The compaction summary is a user message carrying Claude Code's preamble. + let request = openai_chat_request(json!({ + "messages": [ + {"role": "user", "content": "This session is being continued from a previous conversation that ran out of context."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(ToolSignals::from_request(&request, None).compacted); + } + + #[test] + fn no_compaction_marker_stays_uncompacted() { + let request = openai_chat_request(json!({ + "messages": [ + {"role": "user", "content": "Write a script that parses the log file."}, + {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, + ] + })); + assert!(!ToolSignals::from_request(&request, None).compacted); + } + #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. @@ -983,47 +1019,55 @@ mod tests { #[test] fn tests_passed_detects_pytest_with_failure_block() { // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. - assert!(!detect_tests_passed(&[ - "2 failed, 5 passed in 0.56s".to_string() - ])); + assert!(!detect_tests_passed( + &["2 failed, 5 passed in 0.56s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_cargo_clean_summary() { // Cargo's clean-run summary contains "0 failed" — must not trip the // failure list (regression: previously substring-matched "failed"). - assert!(detect_tests_passed(&[ - "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string() - ])); + assert!(detect_tests_passed( + &["running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_rejects_cargo_real_failure() { // Cargo's actual-failure summary: nonzero count before "failed". - assert!(!detect_tests_passed(&[ - "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string() - ])); + assert!(!detect_tests_passed( + &["running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_go_clean_summary() { // Go test's clean-run "0 errors" must not trip (regression). - assert!(detect_tests_passed(&[ - "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string() - ])); + assert!(detect_tests_passed( + &["ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_accepts_pytest_zero_errors() { // Pytest long-form: "0 errors in 0.3s" on a clean run. - assert!(detect_tests_passed(&[ - "5 passed, 0 errors in 0.30s".to_string() - ])); + assert!(detect_tests_passed( + &["5 passed, 0 errors in 0.30s".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] fn tests_passed_detects_diy_checkmark() { - assert!(detect_tests_passed(&["✓ all checks passed".to_string()])); + assert!(detect_tests_passed( + &["✓ all checks passed".to_string()], + DEFAULT_RECENT_WINDOW + )); } #[test] diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index bf5441c2..f0440be5 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -1,1224 +1,47 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Tool-result context signals extracted from the conversation history. +//! Tool-result context signals — thin adapter over libsy's extractor. //! -//! The [`DimensionCollector`][`crate::DimensionCollector`] runs this -//! alongside the 15 prompt-text scorers: it walks the `messages[]` / -//! `input[]` array of the incoming request, finds tool-execution results, -//! pattern-matches their text against a curated error table, and aggregates -//! conversation-history metrics that the stage_router pickers need. -//! -//! All logic is pure and deterministic — no I/O, no shared state. +//! The extraction logic lives in [`switchyard_libsy::ToolSignals`]. This module +//! bridges the crate's [`ChatRequest`] (a format-tagged JSON body) to libsy's +//! [`switchyard_protocol::Request`] (raw body + wire-format metadata) so the two +//! request models share one implementation. -use serde_json::Value; use switchyard_core::{ChatRequest, ChatRequestType}; - -// ─── severity constants ─────────────────────────────────────────────────────── - -const SOFT: f32 = 0.3; -const HARD: f32 = 0.7; -const CRITICAL: f32 = 1.0; - -// ─── pattern table ──────────────────────────────────────────────────────────── - -/// (name, severity, lower-cased substrings — any hit fires the pattern) -static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[ - ( - "oom", - CRITICAL, - &["out of memory", "memoryerror", "cannot allocate memory"], - ), - ( - "connection_refused", - CRITICAL, - &[ - "connection refused", - "connectionrefusederror", - "econnrefused", - ], - ), - ("traceback", HARD, &["traceback (most recent call last)"]), - ( - "import_error", - HARD, - &["modulenotfounderror:", "importerror:", "no module named "], - ), - ( - "cmd_not_found", - HARD, - &["command not found", "not found\n", "/usr/bin/env: "], - ), - ("assertion", HARD, &["assertionerror"]), - ("value_error", HARD, &["valueerror:"]), - ("syntax_error", HARD, &["syntaxerror:"]), - ( - "timeout", - HARD, - &[ - "timed out", - "timeouterror", - "timeout expired", - "deadline exceeded", - ], - ), - ( - "no_such_file", - HARD, - &[ - "filenotfounderror:", - "no such file or directory", - // Claude Code Read-tool miss. Anchored as "file does not exist" (not a - // bare "does not exist", which fires on `ls` output and prose) — trace- - // mined across 1006 local trajectories at 22 true / 2 false positives. - "file does not exist", - ], - ), - // SOFT: plain non-zero exit without a recognisable exception traceback. - ( - "exit_nonzero", - SOFT, - &[ - "exit code 1", - "exit code 2", - "exit status 1", - "returned non-zero", - "exited with code", - ], - ), -]; - -static EDIT_TOOL_NAMES: &[&str] = &[ - "edit", - "multiedit", - "notebookedit", - "str_replace", - "str_replace_based_edit_tool", - "text_editor", -]; - -static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file"]; - -// Bash subcommand patterns. Lowercased; callers must lowercase the command -// before matching. Bucketed into write_count / edit_count alongside the -// dedicated `Write` / `Edit` tools. -static BASH_WRITE_PATTERNS: &[&str] = &[ - "cat >", - "cat >>", - "echo >", - "echo >>", - "tee ", - "printf >", - "printf >>", - "> /", - ">> /", - "<< 'eof'", - "<, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ToolCategory { - Write, - Edit, - Read, - Plan, - Other, -} - -fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { - let lower = name.to_lowercase(); - if WRITE_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Write; - } - if EDIT_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Edit; - } - if READ_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Read; - } - if PLAN_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Plan; - } - if BASH_TOOL_NAMES.contains(&lower.as_str()) { - if let Some(cmd) = command { - // Write/edit redirection trumps read-like operands. - if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Write; - } - if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Edit; - } - if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Read; - } - } +use switchyard_protocol::{Metadata, Request, WireFormat}; + +/// The tool-signal output type. Re-exported from libsy so downstream consumers +/// (the `ToolResultSignal` stamped on `ProxyContext`) see a single type. +pub use switchyard_libsy::{ToolSignals as ToolResultSignal, DEFAULT_RECENT_WINDOW}; + +/// Adapt a format-tagged [`ChatRequest`] to a [`switchyard_protocol::Request`] +/// carrying the raw body and its wire format, which is all libsy's extractor reads. +fn to_protocol_request(request: &ChatRequest) -> Request { + let wire_format = match request.request_type() { + ChatRequestType::OpenAiChat => WireFormat::OpenAiChat, + ChatRequestType::Anthropic => WireFormat::AnthropicMessages, + ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, + }; + Request { + raw_request: Some(request.body().clone()), + metadata: Some(Metadata { + wire_format: Some(wire_format), + ..Default::default() + }), + ..Default::default() } - ToolCategory::Other } -// ─── extraction entry point ─────────────────────────────────────────────────── - -/// Extract all tool-execution signals from a [`ChatRequest`]. -/// -/// Uses [`DEFAULT_RECENT_WINDOW`] for the sliding-window `recent_*` counts. -/// Callers wanting a different window should use -/// [`extract_tool_signals_with_window`] directly. -/// -/// Dispatches on the request's wire format: -/// -/// * **OpenAI chat** — `messages[]` with `role: "tool"` for results; -/// `role: "assistant"` with `tool_calls[]` for call names. -/// * **Anthropic** — `messages[]` with `role: "user"` + `content[].type: -/// "tool_result"`; `role: "assistant"` with `content[].type: "tool_use"`. -/// * **OpenAI responses** — `input[]` with `type: "function_call_output"`; -/// `type: "function_call"` for call names. -/// -/// Returns [`ToolResultSignal::default()`] when the body is absent or -/// the messages list is empty — callers can always read `signal.severity`. +/// Extract tool-execution signals using the default `recent_*` window. pub fn extract_tool_signals(request: &ChatRequest) -> ToolResultSignal { - extract_tool_signals_with_window(request, DEFAULT_RECENT_WINDOW) + ToolResultSignal::from_request(&to_protocol_request(request), None) } -/// Like [`extract_tool_signals`] but with a caller-supplied sliding-window -/// size for the `recent_*` counts. +/// Extract tool-execution signals with a caller-supplied `recent_*` window. pub fn extract_tool_signals_with_window( request: &ChatRequest, recent_window: usize, ) -> ToolResultSignal { - let body = request.body(); - let Some(obj) = body.as_object() else { - return ToolResultSignal::default(); - }; - - let (mut signal, entries) = match request.request_type() { - ChatRequestType::Anthropic => { - let messages = obj - .get("messages") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - ( - extract_from_messages_anthropic(messages, recent_window), - messages, - ) - } - ChatRequestType::OpenAiResponses => { - let items = obj - .get("input") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - (extract_from_input_responses(items, recent_window), items) - } - ChatRequestType::OpenAiChat => { - let messages = obj - .get("messages") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - ( - extract_from_messages_openai_chat(messages, recent_window), - messages, - ) - } - }; - - // Compaction is detected anywhere in the message/item contents (the summary stays - // in the prefix on every subsequent turn, so this self-latches once it fires). - signal.compacted = entries.iter().any(|m| { - m.as_object() - .is_some_and(|o| content_has_compaction_marker(o.get("content"))) - }); - signal -} - -// ─── format-specific extractors ────────────────────────────────────────────── - -/// Distinctive preamble Claude Code injects as a user message when it compacts an -/// overflowed context. Matched case-insensitively; normal task text never contains it. -const COMPACTION_MARKER: &str = "session is being continued"; - -/// True when a user-message content (string or text blocks) carries the compaction -/// summary preamble. -fn content_has_compaction_marker(content: Option<&Value>) -> bool { - match content { - Some(Value::String(s)) => s.to_lowercase().contains(COMPACTION_MARKER), - Some(Value::Array(blocks)) => blocks.iter().any(|b| { - b.as_object() - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - .is_some_and(|t| t.to_lowercase().contains(COMPACTION_MARKER)) - }), - _ => false, - } -} - -fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolResultSignal { - let mut tool_texts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); - - for msg in messages { - let Some(obj) = msg.as_object() else { continue }; - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); - match role { - "tool" => { - if let Some(text) = content_to_text(obj.get("content")) { - tool_texts.push(text); - } - } - "assistant" => { - if let Some(tc_list) = obj.get("tool_calls").and_then(Value::as_array) { - for tc in tc_list { - let Some(fn_obj) = tc - .as_object() - .and_then(|t| t.get("function")) - .and_then(|f| f.as_object()) - else { - continue; - }; - let Some(name) = fn_obj.get("name").and_then(Value::as_str) else { - continue; - }; - // OpenAI Chat encodes `arguments` as a JSON string. - let command = fn_obj - .get("arguments") - .and_then(Value::as_str) - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("command") - .and_then(Value::as_str) - .map(|s| s.to_lowercase()) - }); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); - } - } - } - _ => {} - } - } - - build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) -} - -fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolResultSignal { - let mut tool_texts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); - - for msg in messages { - let Some(obj) = msg.as_object() else { continue }; - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); - let content = obj.get("content"); - - match role { - "user" => { - if let Some(Value::Array(blocks)) = content { - for block in blocks { - let Some(b) = block.as_object() else { continue }; - if b.get("type").and_then(Value::as_str) == Some("tool_result") { - if let Some(text) = content_to_text(b.get("content")) { - tool_texts.push(text); - } - } - } - } - } - "assistant" => { - if let Some(Value::Array(blocks)) = content { - for block in blocks { - let Some(b) = block.as_object() else { continue }; - if b.get("type").and_then(Value::as_str) == Some("tool_use") { - let Some(name) = b.get("name").and_then(Value::as_str) else { - continue; - }; - // Anthropic delivers `input` as a parsed object. - let command = b - .get("input") - .and_then(Value::as_object) - .and_then(|i| i.get("command")) - .and_then(Value::as_str) - .map(|s| s.to_lowercase()); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); - } - } - } - } - _ => {} - } - } - - build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) -} - -fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolResultSignal { - let mut tool_texts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); - - for item in items { - let Some(obj) = item.as_object() else { - continue; - }; - let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - - match item_type { - "function_call_output" => { - if let Some(output) = obj.get("output").and_then(Value::as_str) { - tool_texts.push(output.to_string()); - } - } - "function_call" => { - let Some(name) = obj.get("name").and_then(Value::as_str) else { - continue; - }; - let command = obj - .get("arguments") - .and_then(Value::as_str) - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("command") - .and_then(Value::as_str) - .map(|s| s.to_lowercase()) - }); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); - } - _ => {} - } - } - - build_signal(tool_texts, tool_calls, items.len() as u32, recent_window) -} - -// ─── aggregation ───────────────────────────────────────────────────────────── - -fn build_signal( - tool_texts: Vec, - tool_calls: Vec, - turn_depth: u32, - recent_window: usize, -) -> ToolResultSignal { - // Windowed severity: take the MAX severity across the last `recent_window` tool - // results rather than only the last one. An error's severity then persists for - // the recent window and decays out of it — parallel to the windowed `recent_*` - // counts — so a fix written a couple of turns after an error still routes on the - // error signal instead of the router flapping straight back to the weak tier. - let sev_start = tool_texts.len().saturating_sub(recent_window.max(1)); - let mut severity = 0.0f32; - for text in &tool_texts[sev_start..] { - let (sev, _patterns) = classify_text(text); - if sev > severity { - severity = sev; - } - } - - let no_error_streak = compute_no_error_streak(&tool_texts); - - // Single pass: cumulative + sliding-window counters together. Also tracks - // the trailing pure-bash streak (consecutive `Other`-category calls back - // from the end) — the build-pit proxy. - let recent_start = tool_calls.len().saturating_sub(recent_window); - let mut write_count = 0u32; - let mut edit_count = 0u32; - let mut read_count = 0u32; - let mut todowrite_count = 0u32; - let mut recent_write_count = 0u32; - let mut recent_edit_count = 0u32; - let mut recent_read_count = 0u32; - let mut recent_todowrite_count = 0u32; - let mut pure_bash_streak = 0u32; - let mut streak_open = true; - for (i, tc) in tool_calls.iter().enumerate().rev() { - let cat = classify_tool_call(&tc.name, tc.command.as_deref()); - if streak_open { - if matches!(cat, ToolCategory::Other) { - pure_bash_streak += 1; - } else { - streak_open = false; - } - } - match cat { - ToolCategory::Write => { - write_count += 1; - if i >= recent_start { - recent_write_count += 1; - } - } - ToolCategory::Edit => { - edit_count += 1; - if i >= recent_start { - recent_edit_count += 1; - } - } - ToolCategory::Read => { - read_count += 1; - if i >= recent_start { - recent_read_count += 1; - } - } - ToolCategory::Plan => { - todowrite_count += 1; - if i >= recent_start { - recent_todowrite_count += 1; - } - } - ToolCategory::Other => {} - } - } - - let tests_passed = detect_tests_passed(&tool_texts, recent_window); - - ToolResultSignal { - severity, - no_error_streak, - edit_count, - write_count, - read_count, - todowrite_count, - recent_edit_count, - recent_write_count, - recent_read_count, - recent_todowrite_count, - pure_bash_streak, - tests_passed, - turn_depth, - // Set by extract_tool_signals_with_window after the format-specific extract, - // which scans all message contents for the compaction marker. - compacted: false, - } -} - -// ─── pure helpers ───────────────────────────────────────────────────────────── - -/// Normalise a JSON tool-result content value to a plain string. -fn content_to_text(content: Option<&Value>) -> Option { - match content? { - Value::String(s) => Some(s.clone()), - Value::Array(blocks) => { - let parts: Vec<&str> = blocks - .iter() - .filter_map(|b| { - b.as_object() - .filter(|o| o.get("type").and_then(Value::as_str) == Some("text")) - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - }) - .collect(); - if parts.is_empty() { - None - } else { - Some(parts.join("\n")) - } - } - _ => None, - } -} - -/// Match `text` against the error pattern table. -/// -/// Returns `(max_severity, matched_pattern_names)`. -pub(crate) fn classify_text(text: &str) -> (f32, Vec) { - let lower = text.to_lowercase(); - let mut patterns = Vec::new(); - let mut severity: f32 = 0.0; - for (name, sev, substrings) in ERROR_PATTERNS { - if substrings.iter().any(|sub| lower.contains(sub)) { - patterns.push(name.to_string()); - severity = severity.max(*sev); - } - } - (severity, patterns) -} - -fn compute_no_error_streak(tool_texts: &[String]) -> u32 { - let mut streak = 0u32; - for text in tool_texts.iter().rev() { - let (sev, _) = classify_text(text); - if sev > 0.0 { - break; - } - streak += 1; - } - streak -} - -fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool { - let start = tool_texts.len().saturating_sub(recent_window.max(1)); - tool_texts[start..].iter().any(|text| { - let lower = text.to_lowercase(); - TEST_PASS_PHRASES.iter().any(|p| lower.contains(p)) - && !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p)) - && !has_nonzero_failure_count(&lower) - }) -} - -// True iff `lower` contains a `NUMERIC_FAILURE_KEYWORDS` token preceded -// (modulo whitespace) by a nonzero integer. The "modulo whitespace" lets -// "1 failed", "1\nfailed", and "1 failed" all trip; the nonzero guard -// keeps cargo's "0 failed" / go's "0 errors" / pytest's "0 errors in" -// summaries from being misread as failures on a clean run. -fn has_nonzero_failure_count(lower: &str) -> bool { - for kw in NUMERIC_FAILURE_KEYWORDS { - let mut cursor = 0usize; - while let Some(rel) = lower[cursor..].find(kw) { - let kw_start = cursor + rel; - let kw_end = kw_start + kw.len(); - // Word boundary AFTER the keyword — "errors" mid-word (e.g. - // "errored") shouldn't count as a failure-count site. - let boundary_after = lower[kw_end..] - .chars() - .next() - .is_none_or(|c| !c.is_ascii_alphanumeric()); - if boundary_after { - let prefix = &lower[..kw_start]; - let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace()); - let digits_rev: String = trimmed - .chars() - .rev() - .take_while(|c| c.is_ascii_digit()) - .collect(); - if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') { - return true; - } - } - cursor = kw_start + kw.len(); - } - } - false -} - -// ─── tests ─────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn clean_text_has_zero_severity() { - let (sev, patterns) = classify_text("everything went fine"); - assert_eq!(sev, 0.0); - assert!(patterns.is_empty()); - } - - #[test] - fn traceback_is_hard() { - let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError"); - assert_eq!(sev, HARD); - assert!(patterns.contains(&"traceback".to_string())); - } - - #[test] - fn oom_is_critical() { - let (sev, _) = classify_text("Out of memory: kill process 1234"); - assert_eq!(sev, CRITICAL); - } - - #[test] - fn severity_is_max_across_patterns() { - // exit_nonzero (SOFT) + traceback (HARD) → HARD. - let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):"); - assert_eq!(sev, HARD); - } - - #[test] - fn file_does_not_exist_is_hard() { - // Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives). - let (sev, patterns) = - classify_text("Error: File does not exist. Note: current working directory is /app."); - assert_eq!(sev, HARD); - assert!(patterns.contains(&"no_such_file".to_string())); - } - - #[test] - fn bare_does_not_exist_stays_clean() { - // Precision guard: only the anchored "file does not exist" fires, so a bare - // "does not exist" in prose or directory output must not trip a false error. - let (sev, _) = classify_text("The directory does not exist yet, creating it now."); - assert_eq!(sev, 0.0); - } - - #[test] - fn no_error_streak_all_clean() { - let texts = vec!["ok".to_string(), "all good".to_string()]; - assert_eq!(compute_no_error_streak(&texts), 2); - } - - #[test] - fn no_error_streak_stops_at_error() { - let texts = vec![ - "Traceback (most recent call last):".to_string(), - "ok".to_string(), - "ok".to_string(), - ]; - assert_eq!(compute_no_error_streak(&texts), 2); - } - - #[test] - fn tests_passed_detects_pytest_output() { - assert!(detect_tests_passed( - &["====== 5 passed in 0.12s ======".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_ignores_partial_failures() { - assert!(!detect_tests_passed( - &["2 failed, 5 passed in 0.56s".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn severity_is_windowed_over_recent_results() { - // An error two results back, then two clean results. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "tool", "tool_call_id": "1", - "content": "Traceback (most recent call last):\n ValueError"}, - {"role": "tool", "tool_call_id": "2", "content": "ok"}, - {"role": "tool", "tool_call_id": "3", "content": "ok"}, - ] - })); - // window covers the error → severity persists (max over the window) - assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD); - // window of 1 sees only the last (clean) result → severity has decayed out - assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0); - } - - #[test] - fn extract_openai_chat_tool_results() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "user", "content": "do something"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "tool_call_id": "1", "content": "Traceback (most recent call last):\n ValueError"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.severity, HARD); - assert_eq!(sig.edit_count, 1); - assert_eq!(sig.turn_depth, 3); - } - - #[test] - fn extract_anthropic_tool_results() { - let request = ChatRequest::anthropic(json!({ - "messages": [ - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "1", - "content": "Traceback (most recent call last):\n ValueError"} - ]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.severity, HARD); - } - - #[test] - fn extract_responses_api_tool_results() { - let request = ChatRequest::openai_responses(json!({ - "input": [ - {"type": "function_call", "name": "Write"}, - {"type": "function_call_output", "call_id": "1", - "output": "file written successfully"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.severity, 0.0); - assert_eq!(sig.write_count, 1); - } - - #[test] - fn recent_window_counts_only_last_default_window_tool_calls() { - // 5 writes + 1 edit at the end → the default window (5) should see - // the last 5 calls: 1 edit + 4 writes (not all 6 calls). - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.write_count, 5); - assert_eq!(sig.edit_count, 1); - assert_eq!(sig.recent_write_count, 4); - assert_eq!(sig.recent_edit_count, 1); - } - - #[test] - fn recent_window_size_is_caller_overridable() { - // Same six tool calls (1 edit at the end, 5 writes before). - // With recent_window=3 → recent_writes=2, recent_edits=1. - // With recent_window=6 → recent_writes=5, recent_edits=1 (all calls). - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); - let narrow = extract_tool_signals_with_window(&request, 3); - assert_eq!(narrow.recent_write_count, 2); - assert_eq!(narrow.recent_edit_count, 1); - - let wide = extract_tool_signals_with_window(&request, 6); - assert_eq!(wide.recent_write_count, 5); - assert_eq!(wide.recent_edit_count, 1); - } - - #[test] - fn compaction_marker_sets_compacted() { - // The compaction summary is a user message carrying Claude Code's preamble. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "user", "content": "This session is being continued from a previous conversation that ran out of context."}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, - ] - })); - assert!(extract_tool_signals(&request).compacted); - } - - #[test] - fn no_compaction_marker_stays_uncompacted() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "user", "content": "Write a script that parses the log file."}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, - ] - })); - assert!(!extract_tool_signals(&request).compacted); - } - - #[test] - fn bash_heredoc_counts_as_write() { - // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": { - "name": "Bash", - "arguments": "{\"command\": \"cat > /tmp/test.py <<'EOF'\\nprint(1)\\nEOF\"}" - } - }]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!( - sig.write_count, 1, - "Bash heredoc should bucket into write_count" - ); - assert_eq!(sig.edit_count, 0); - } - - #[test] - fn bash_sed_inplace_counts_as_edit() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": { - "name": "Bash", - "arguments": "{\"command\": \"sed -i 's/foo/bar/g' /app/file.py\"}" - } - }]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!( - sig.edit_count, 1, - "Bash sed -i should bucket into edit_count" - ); - assert_eq!(sig.write_count, 0); - } - - #[test] - fn bash_non_mutating_does_not_count() { - // ls, cat, grep — should not increment either counter. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": {"name": "Bash", "arguments": "{\"command\": \"ls -la /app\"}"} - }]}, - {"role": "assistant", "tool_calls": [{ - "function": {"name": "Bash", "arguments": "{\"command\": \"cat /app/main.py\"}"} - }]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.write_count, 0); - assert_eq!(sig.edit_count, 0); - } - - #[test] - fn tests_passed_detects_pytest_with_failure_block() { - // Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed. - assert!(!detect_tests_passed( - &["2 failed, 5 passed in 0.56s".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_accepts_cargo_clean_summary() { - // Cargo's clean-run summary contains "0 failed" — must not trip the - // failure list (regression: previously substring-matched "failed"). - assert!(detect_tests_passed( - &["running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_rejects_cargo_real_failure() { - // Cargo's actual-failure summary: nonzero count before "failed". - assert!(!detect_tests_passed( - &["running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_accepts_go_clean_summary() { - // Go test's clean-run "0 errors" must not trip (regression). - assert!(detect_tests_passed( - &["ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_accepts_pytest_zero_errors() { - // Pytest long-form: "0 errors in 0.3s" on a clean run. - assert!(detect_tests_passed( - &["5 passed, 0 errors in 0.30s".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn tests_passed_detects_diy_checkmark() { - assert!(detect_tests_passed( - &["✓ all checks passed".to_string()], - DEFAULT_RECENT_WINDOW - )); - } - - #[test] - fn anthropic_bash_heredoc_extracts_command() { - // Anthropic format: tool_use.input is an object, not a JSON string. - let request = ChatRequest::anthropic(json!({ - "messages": [ - {"role": "assistant", "content": [ - {"type": "tool_use", "name": "Bash", - "input": {"command": "cat > /tmp/foo.txt << 'EOF'\nhi\nEOF"}} - ]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!( - sig.write_count, 1, - "Anthropic Bash heredoc must also be detected" - ); - } - - #[test] - fn recent_window_falls_back_to_full_history_when_short() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.recent_write_count, 1); - assert_eq!(sig.recent_edit_count, 0); - } - - #[test] - fn clean_tool_result_has_zero_severity_and_non_empty_streak() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "tool", "tool_call_id": "1", "content": "output ok"}, - {"role": "tool", "tool_call_id": "2", "content": "another ok"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.severity, 0.0); - assert_eq!(sig.no_error_streak, 2); - } - - // ─── asymmetric-signal extensions ──────────────────────────────────── - - #[test] - fn todowrite_classifies_as_plan() { - assert_eq!(classify_tool_call("TodoWrite", None), ToolCategory::Plan); - assert_eq!(classify_tool_call("todo_write", None), ToolCategory::Plan); - } - - #[test] - fn codex_update_plan_classifies_as_plan() { - assert_eq!(classify_tool_call("update_plan", None), ToolCategory::Plan); - } - - #[test] - fn codex_shell_command_runs_bash_pattern_match() { - // shell_command + heredoc -> Write. - assert_eq!( - classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")), - ToolCategory::Write, - ); - // shell_command + read-like inspection -> Read. - assert_eq!( - classify_tool_call("shell_command", Some("ls /app")), - ToolCategory::Read, - ); - // shell_command without matching patterns -> Other. - assert_eq!( - classify_tool_call("shell_command", Some("./run_tests.sh")), - ToolCategory::Other, - ); - } - - #[test] - fn read_tool_classifies_as_read() { - assert_eq!(classify_tool_call("Read", None), ToolCategory::Read); - assert_eq!(classify_tool_call("View", None), ToolCategory::Read); - } - - #[test] - fn bash_read_patterns_classify_as_read() { - let cases = [ - "cat /etc/passwd", - "grep foo bar.txt", - "ls /app", - "find . -name '*.py'", - ]; - for cmd in cases { - assert_eq!( - classify_tool_call("Bash", Some(cmd)), - ToolCategory::Read, - "expected Read for {cmd}" - ); - } - } - - #[test] - fn bash_write_precedence_over_read() { - // `cat /file > out` contains both `cat /` (read) and ` > ` (write); - // write redirection must win. - assert_eq!( - classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")), - ToolCategory::Write, - ); - } - - #[test] - fn pure_bash_streak_counts_trailing_other() { - // 5 trailing non-classified Bash calls → streak == 5. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./configure\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make install\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./run.sh\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./test\"}"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.pure_bash_streak, 5); - assert_eq!(sig.write_count, 0); - assert_eq!(sig.read_count, 0); - } - - #[test] - fn pure_bash_streak_resets_on_write() { - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.pure_bash_streak, 0); - assert_eq!(sig.write_count, 1); - } - - #[test] - fn recent_window_tracks_todowrite_and_read() { - // Final 3 tool calls: TodoWrite, Read, TodoWrite. - let request = ChatRequest::openai_chat(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Read"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); - let sig = extract_tool_signals(&request); - assert_eq!(sig.todowrite_count, 2); - assert_eq!(sig.recent_todowrite_count, 2); - assert_eq!(sig.read_count, 1); - assert_eq!(sig.recent_read_count, 1); - } + ToolResultSignal::from_request(&to_protocol_request(request), Some(recent_window)) } From 2f7ba8807fd9419b902b7f96e773b6faf9ee8019 Mon Sep 17 00:00:00 2001 From: Greg Clark Date: Thu, 23 Jul 2026 20:45:43 -0400 Subject: [PATCH 18/21] chore: formating Signed-off-by: Greg Clark --- .../switchyard-components/src/stage_router.rs | 27 +++++++++++++++---- .../src/component_bindings/stage_router.rs | 4 +-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs index f8171137..6cbee219 100644 --- a/crates/switchyard-components/src/stage_router.rs +++ b/crates/switchyard-components/src/stage_router.rs @@ -265,7 +265,12 @@ pub fn pick_tier( } else { Tier::Efficient }; - return resolved(tier, DecisionSource::Dimensions, scored.score, Some(scored.confidence)); + return resolved( + tier, + DecisionSource::Dimensions, + scored.score, + Some(scored.confidence), + ); } // 4. Fall open — the signals didn't corroborate enough to be sure. Hand off @@ -278,7 +283,12 @@ pub fn pick_tier( } /// Build a resolved outcome (a decision made without the classifier). -fn resolved(tier: Tier, source: DecisionSource, score: f64, confidence: Option) -> PickOutcome { +fn resolved( + tier: Tier, + source: DecisionSource, + score: f64, + confidence: Option, +) -> PickOutcome { PickOutcome::Resolved { tier, source, @@ -299,8 +309,8 @@ fn ratio(numerator: u32, denominator: u32) -> f64 { mod tests { use super::*; use crate::dimension_collector::extract_tool_signals; - use switchyard_core::ChatRequest; use serde_json::json; + use switchyard_core::ChatRequest; fn signal_from(messages: serde_json::Value) -> ToolResultSignal { let request = ChatRequest::openai_chat(json!({"model": "m", "messages": messages})); @@ -326,7 +336,11 @@ mod tests { signal.compacted = true; assert!(matches!( pick_tier(&signal, PickerMode::EfficientFirst, 0.5), - PickOutcome::Resolved { tier: Tier::Capable, source: DecisionSource::Override, .. } + PickOutcome::Resolved { + tier: Tier::Capable, + source: DecisionSource::Override, + .. + } )); } @@ -337,7 +351,10 @@ mod tests { signal.severity = HARD_SEVERITY as f32; let scored = score_signal(&signal); assert!(scored.score > 0.0); - assert!(scored.confidence < 0.5, "one signal should not clear 0.5: {scored:?}"); + assert!( + scored.confidence < 0.5, + "one signal should not clear 0.5: {scored:?}" + ); } #[test] diff --git a/crates/switchyard-py/src/component_bindings/stage_router.rs b/crates/switchyard-py/src/component_bindings/stage_router.rs index b8795e62..95449a16 100644 --- a/crates/switchyard-py/src/component_bindings/stage_router.rs +++ b/crates/switchyard-py/src/component_bindings/stage_router.rs @@ -12,9 +12,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use switchyard_components::stage_router::{ - pick_tier, score_signal, PickOutcome, PickerMode, Tier, -}; +use switchyard_components::stage_router::{pick_tier, score_signal, PickOutcome, PickerMode, Tier}; use super::dimension_collector::PyToolResultSignal; From d90b832e9bd2ce69fb8386902b80cc531b3211ea Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 24 Jul 2026 09:17:07 -0700 Subject: [PATCH 19/21] fix(ci): SPDX on score_staged_run, sync .pyi stub, narrow picker types, drop patterns test assertions Signed-off-by: Sabhatina Selvam --- benchmark/score_staged_run.py | 2 + .../lib/processors/stage_router/picker.py | 6 ++- switchyard_rust/components.pyi | 54 ++++++------------- tests/test_tool_result_signal_collector.py | 3 -- 4 files changed, 23 insertions(+), 42 deletions(-) diff --git a/benchmark/score_staged_run.py b/benchmark/score_staged_run.py index 988e2d59..7bfafba7 100644 --- a/benchmark/score_staged_run.py +++ b/benchmark/score_staged_run.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 """Score a benchmark run directory via the stage-router Rust scorer. Reads trajectory.json from each completed task, feeds each tool-use turn through: diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py index 66028fc3..c144cf3e 100644 --- a/switchyard/lib/processors/stage_router/picker.py +++ b/switchyard/lib/processors/stage_router/picker.py @@ -10,7 +10,7 @@ classifier, and decision-source recording.""" import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from switchyard.lib.processors.stage_router.decision_log import ( CONTEXT_KEY, @@ -71,7 +71,9 @@ async def _pick( # The Rust core runs the escalate/de-escalate shortcuts and the scorer. outcome = stage_pick_tier(signal, picker_mode, confidence_threshold) if outcome.resolved: - return _record(ctx, decision_log, outcome.source, _TIER[outcome.tier]) + # resolved ⇒ tier and source are set (guaranteed by the Rust core). + source = cast("DecisionSource", outcome.source) + return _record(ctx, decision_log, source, _TIER[cast(str, outcome.tier)]) # Scorer wasn't confident. Consult the classifier; fall open to the default # tier when there is none or it can't decide. diff --git a/switchyard_rust/components.pyi b/switchyard_rust/components.pyi index 7facbbf7..dc2b76a6 100644 --- a/switchyard_rust/components.pyi +++ b/switchyard_rust/components.pyi @@ -286,40 +286,9 @@ class IntakeResponseProcessor: async def shutdown(self) -> None: ... -class DimensionScore: - name: str - score: float - signal: str | None - - -class ContextSignals: - dimensions: list[DimensionScore] - token_count_estimate: int - - -class ScoringConfig: - def __init__( - self, - token_count_short: int = 50, - token_count_long: int = 500, - code_keywords: Iterable[str] = (), - reasoning_keywords: Iterable[str] = (), - simple_keywords: Iterable[str] = (), - technical_keywords: Iterable[str] = (), - creative_keywords: Iterable[str] = (), - imperative_verbs: Iterable[str] = (), - constraint_indicators: Iterable[str] = (), - output_format_keywords: Iterable[str] = (), - reference_keywords: Iterable[str] = (), - negation_keywords: Iterable[str] = (), - domain_specific_keywords: Iterable[str] = (), - ) -> None: ... - - class DimensionCollector: def __init__( self, - config: ScoringConfig | None = None, *, recent_window: int | None = None, ) -> None: ... @@ -328,12 +297,8 @@ class DimensionCollector: async def shutdown(self) -> None: ... -def get_context_signals(ctx: ProxyContext) -> ContextSignals | None: ... - - class ToolResultSignal: severity: float - patterns: tuple[str, ...] turn_depth: int write_count: int edit_count: int @@ -346,14 +311,29 @@ class ToolResultSignal: pure_bash_streak: int no_error_streak: int tests_passed: bool - prompt_char_count: int - repeated_cmd_ratio: float compacted: bool def get_tool_result_signal(ctx: ProxyContext) -> ToolResultSignal | None: ... +class PickOutcome: + resolved: bool + tier: str | None + source: str | None + default_tier: str + score: float + confidence: float | None + + +def stage_pick_tier( + signal: ToolResultSignal, picker_mode: str, confidence_threshold: float +) -> PickOutcome: ... + + +def stage_score_signal(signal: ToolResultSignal) -> tuple[float, float]: ... + + class ResponseFlag: MALFORMED_TOOL_CALL_JSON: ClassVar[ResponseFlag] EMPTY_RESPONSE: ClassVar[ResponseFlag] diff --git a/tests/test_tool_result_signal_collector.py b/tests/test_tool_result_signal_collector.py index 8443ca37..5c584647 100644 --- a/tests/test_tool_result_signal_collector.py +++ b/tests/test_tool_result_signal_collector.py @@ -42,7 +42,6 @@ async def test_traceback_stamps_hard_severity(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.7) - assert "traceback" in list(signal.patterns) async def test_oom_stamps_critical_severity(): @@ -65,7 +64,6 @@ async def test_clean_result_has_zero_severity(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.0) - assert list(signal.patterns) == [] async def test_no_tool_results_has_zero_severity(): @@ -175,7 +173,6 @@ async def test_anthropic_tool_result_extracted(): signal = get_tool_result_signal(ctx) assert signal is not None assert signal.severity == pytest.approx(0.7) - assert "import_error" in list(signal.patterns) # ─── OpenAI Responses format ────────────────────────────────────────────────── From f6aa5220dd7f78f151a61db02f98d8290defc714 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 24 Jul 2026 09:32:41 -0700 Subject: [PATCH 20/21] =?UTF-8?q?docs(stage-router):=20sync=20docs=20+=20s?= =?UTF-8?q?corer=20skill=20to=20the=20Rust=20core=20(score=5Frun=E2=86=92s?= =?UTF-8?q?core=5Fstaged=5Frun,=20stage=5Fscore=5Fsignal,=20escalate/de-es?= =?UTF-8?q?calate)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sabhatina Selvam --- .../skills/switchyard-stage-router-scorer/SKILL.md | 13 +++++++------ docs/routing_algorithms/stage_router_routing.md | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.agents/skills/switchyard-stage-router-scorer/SKILL.md b/.agents/skills/switchyard-stage-router-scorer/SKILL.md index 01e57e6f..01a6718f 100644 --- a/.agents/skills/switchyard-stage-router-scorer/SKILL.md +++ b/.agents/skills/switchyard-stage-router-scorer/SKILL.md @@ -6,23 +6,23 @@ | Script | Purpose | Input | Output | |--------|---------|-------|--------| -| `benchmark/score_run.py` | Score a live run dir via real picker | run dir path | per-turn JSONL + per-task CSV | +| `benchmark/score_staged_run.py` | Score a live run dir via real picker | run dir path | per-turn JSONL + per-task CSV | ## Quick Reference ```bash # Score a run -uv run python benchmark/score_run.py --run benchmark/tb_runs/ +uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ # → /tmp/-scores.jsonl (per turn) # → /tmp/-per-task.csv (per task) # Custom threshold or window -uv run python benchmark/score_run.py \ +uv run python benchmark/score_staged_run.py \ --run benchmark/tb_runs/ \ --threshold 0.15 --window 3 ``` -## Scoring pipeline (what score_run.py does per turn) +## Scoring pipeline (what score_staged_run.py does per turn) ``` trajectory step (tool_use + tool_result) @@ -30,7 +30,7 @@ trajectory step (tool_use + tool_result) → ChatRequest.anthropic({"model": ..., "messages": messages}) # Rust binding → dc.process(ctx, request) # DimensionCollector — one per task, accumulates state → get_tool_result_signal(ctx) # read signal from ctx - → from_signal(signal) + scorer_score(dims) # raw score + confidence (for analysis) + → stage_score_signal(signal) # raw (score, confidence) from the Rust scorer (for analysis) → pick_capable_first(ctx, threshold) # actual cf decision (same ctx, no re-process) → pick_efficient_first(ctx, threshold) # actual ef decision (same ctx, no re-process) ``` @@ -39,7 +39,8 @@ trajectory step (tool_use + tool_result) already stored in that ctx — no duplicate processing. **What the picker does beyond raw score:** -- `_apply_overrides`: `severity >= 1.0` → force CAPABLE; `tests_passed AND depth >= 10 AND writes <= 1` → force EFFICIENT +- **escalate** (`should_escalate`): `compacted` OR `severity >= 1.0` → force CAPABLE +- **de-escalate** (`should_deescalate`): `tests_passed AND recent_write+edit >= 1 AND severity <= 0` → force EFFICIENT - `confidence < threshold` → fall_open to default tier (CAPABLE for cf, EFFICIENT for ef) - Only when `confidence >= threshold`: route by score direction diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 32b75882..be8b6dde 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -156,14 +156,14 @@ From the overlap tasks (those with both capable and efficient results): **Running the sweep** Replay your runs through the real Rust scorer and picker with -`benchmark/score_run.py` (the `switchyard-stage-router-scorer` skill). It emits +`benchmark/score_staged_run.py` (the `switchyard-stage-router-scorer` skill). It emits per-turn scores and per-task routing splits at a given threshold and window — the actual `pick_capable_first` / `pick_efficient_first` decisions, not a counterfactual: ```bash # Score a probe run at a candidate threshold -uv run python benchmark/score_run.py --run benchmark/tb_runs/ \ +uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/ \ --threshold 0.5 --window 3 # → /tmp/-scores.jsonl (per turn: score, confidence, pick_cf, pick_ef) # → /tmp/-per-task.csv (per task: routing split, mean score/confidence) From 96fc5e3cfc23e18d87958795773859f8d5e74ae8 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Fri, 24 Jul 2026 09:40:11 -0700 Subject: [PATCH 21/21] refactor(stage_router): score via direct field math (drop string-keyed weights); trim verbose comments Signed-off-by: Sabhatina Selvam --- .../switchyard-components/src/stage_router.rs | 64 +++++-------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs index 6cbee219..cf8c7d83 100644 --- a/crates/switchyard-components/src/stage_router.rs +++ b/crates/switchyard-components/src/stage_router.rs @@ -3,21 +3,15 @@ //! Stage-router scoring and tier selection — the shared routing core. //! -//! Given a [`ToolResultSignal`] (extracted by the dimension collector), this -//! module decides whether a coding-agent turn should go to the **capable** -//! (strong) or **efficient** (weak) tier. It is the single source of truth for -//! the decision, shared by the Rust profile and the Python processor via -//! bindings — only the outer shell differs in how it fetches the decision. +//! Decides whether a coding-agent turn goes to the **capable** (strong) or +//! **efficient** (weak) tier from its [`ToolResultSignal`]. Single source of +//! truth, shared by the Rust profile and the Python processor via bindings — +//! only the outer shell differs in how it fetches the decision. //! -//! Two axes: -//! -//! * **error** — did the recent tool results error? (`severity`) -//! * **production** — is the agent producing code? (`spinning` / `exploring` -//! push toward capable; `production_intensity` pushes toward efficient) -//! -//! Signals are scored with fixed weights, summed, and `tanh`-squashed into -//! `(-1, +1)`; `confidence` is the magnitude. The `confidence_threshold` dials -//! how much corroboration a decisive escalation needs (see [`score_signal`]). +//! Two axes: **error** (`severity`) and **production** (`spinning`/`exploring` +//! push toward capable, `production_intensity` toward efficient). Scored, +//! `tanh`-squashed to `(-1, +1)`, dialed by `confidence_threshold`. See +//! [`pick_tier`] for the decision ladder. use crate::dimension_collector::ToolResultSignal; @@ -37,17 +31,6 @@ const SIGNAL_UNIT: f64 = 0.10; /// Critical severity forces the capable tier regardless of the scorer. const SEVERITY_CRITICAL: f32 = 1.0; -/// Signed, fixed weights over the two axes. Error signals (`severity`, -/// `spinning`, `exploring`) push toward capable (+); `production_intensity` -/// pushes toward efficient (−). `severity` is normalised by its hard cap so it -/// too contributes one `SIGNAL_UNIT` when maxed. -const DEFAULT_WEIGHTS: &[(&str, f64)] = &[ - ("severity", SIGNAL_UNIT / HARD_SEVERITY), - ("spinning", SIGNAL_UNIT), - ("exploring", SIGNAL_UNIT), - ("production_intensity", -SIGNAL_UNIT), -]; - /// The two tiers a turn can route to. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Tier { @@ -122,18 +105,6 @@ pub struct CodingAgentDimensions { pub production_intensity: f64, } -impl CodingAgentDimensions { - fn value(&self, name: &str) -> f64 { - match name { - "severity" => self.severity, - "spinning" => self.spinning, - "exploring" => self.exploring, - "production_intensity" => self.production_intensity, - _ => 0.0, - } - } -} - /// Outcome of [`pick_tier`]: either a resolved decision, or a signal that the /// caller should consult its (impl-specific, async) classifier. #[derive(Clone, Copy, Debug, PartialEq)] @@ -186,18 +157,17 @@ pub fn dimensions_from_signal(signal: &ToolResultSignal) -> CodingAgentDimension } } -/// Score a signal: weighted sum of the dimensions, `tanh`-squashed. +/// Score a signal on the two axes, `tanh`-squashed to `(-1, +1)`. /// -/// The raw sum is small — one maxed signal is `±0.10`, two corroborating -/// signals `±0.20`. `tanh(gain·raw)` spreads that into a usable range, so the -/// `confidence_threshold` reads roughly: `~0.3` escalates on one signal, `~0.5` -/// needs about one-and-a-half, `~0.7` needs two to corroborate. +/// One maxed signal reads ~`0.46` confidence, two ~`0.76` — so the threshold +/// dials corroboration: `~0.3` clears on one signal, `~0.5` on ~1.5, `~0.7` on two. pub fn score_signal(signal: &ToolResultSignal) -> ScoreResult { - let dimensions = dimensions_from_signal(signal); - let raw: f64 = DEFAULT_WEIGHTS - .iter() - .map(|(name, weight)| dimensions.value(name) * weight) - .sum(); + let d = dimensions_from_signal(signal); + // Error axis (severity, spinning, exploring → +capable) minus the production + // axis (→ −efficient). Each maxed signal is one SIGNAL_UNIT; severity is + // normalised by its hard cap so it lands there too. + let raw = SIGNAL_UNIT + * (d.severity / HARD_SEVERITY + d.spinning + d.exploring - d.production_intensity); let score = (SCORE_GAIN * raw).tanh(); ScoreResult { score,