diff --git a/.agents/skills/switchyard-codebase-exploration/SKILL.md b/.agents/skills/switchyard-codebase-exploration/SKILL.md index 165c7e8d..8057b926 100644 --- a/.agents/skills/switchyard-codebase-exploration/SKILL.md +++ b/.agents/skills/switchyard-codebase-exploration/SKILL.md @@ -7,8 +7,8 @@ description: Use when modifying, debugging, reviewing, refactoring, renaming, re > Rust crates under `crates/` are active on the Rust core branch. Inspect > `crates/switchyard-core`, `crates/switchyard-components`, -> `crates/switchyard-components-v2`, `crates/switchyard-server`, -> `crates/switchyard-translation`, `crates/switchyard-py`, and `crates/*/tests` +> `crates/switchyard-server`, `crates/switchyard-translation`, +> `crates/switchyard-py`, and `crates/*/tests` > directly when a change touches Rust. > Direct PyO3 bindings for concrete components live under > `crates/switchyard-py/src/component_bindings/` with Python lazy exports in diff --git a/.agents/skills/switchyard-coding-agent-launchers/SKILL.md b/.agents/skills/switchyard-coding-agent-launchers/SKILL.md index 0c785614..099bb87c 100644 --- a/.agents/skills/switchyard-coding-agent-launchers/SKILL.md +++ b/.agents/skills/switchyard-coding-agent-launchers/SKILL.md @@ -85,8 +85,8 @@ Route YAML and launchers share this model-dispatch path: - `type: passthrough` registers the YAML key and discovered direct models. - `type: random_routing` registers the YAML key as the routing profile plus direct strong/weak passthrough entries. -- `type: deterministic` and `type: stage_router` register the route key plus direct - strong/weak passthrough entries. +- `type: deterministic`, `type: escalation_router`, and `type: stage_router` + register the route key plus direct strong/weak passthrough entries. - `type: plan_execute` registers the route key plus the executor as a direct passthrough; the planner is internal routing logic. - `type: latency_service` and `type: noop` register the diff --git a/AGENTS.md b/AGENTS.md index 3a9f7253..73dba19e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -246,10 +246,8 @@ and their transitives never appear in downstream vulnerability scans. export OPENAI_API_KEY="sk-..." # or NVIDIA_API_KEY / ANTHROPIC_API_KEY where supported export OPENROUTER_API_KEY="sk-or-..." # pass with --api-key or save via configure -# Serve a profile config (passthrough, random-routing, llm-routing, stage_router). -# Endpoints, targets, and profiles live in the YAML; see docs/routing_algorithms/overview.md. -switchyard serve --config profiles.yaml --port 4000 -switchyard serve --config profiles.yaml --inbound anthropic --port 4000 +# Serve a routing bundle. Routes live in YAML; see docs/routing_algorithms/overview.md. +switchyard --routing-profiles routes.yaml -- serve --port 4000 # One-command launchers — single-model passthrough via --model switchyard launch claude --model openai/gpt-4o-mini \ @@ -291,7 +289,7 @@ uv run mypy switchyard 1. Pick the right stage: request component (pre-call), response component (post-call), `LLMBackend` (rare), or Rust translation codec work. 2. Create a file with the explicit name (`snake_case` of the class name), one class per file. 3. Implement the async method for that stage (`process` for components, `call` for backends). -4. Wire into the owning profile config. +4. Wire into the owning programmatic profile config or route-bundle builder. 5. Add tests under `tests/`. 6. Export from the relevant `__init__.py` and from `switchyard/__init__.py`'s `__all__`. diff --git a/Cargo.lock b/Cargo.lock index b10531a5..8129d2df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,12 +912,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "libyaml-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1671,38 +1665,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "switchyard-components-v2" -version = "0.1.0" -dependencies = [ - "async-stream", - "async-trait", - "futures-core", - "futures-util", - "parking_lot", - "rand 0.8.7", - "reqwest", - "serde", - "serde_json", - "switchyard-components", - "switchyard-components-v2-macros", - "switchyard-core", - "switchyard-protocol", - "tokio", - "toml", - "tracing", - "yaml_serde", -] - -[[package]] -name = "switchyard-components-v2-macros" -version = "0.1.0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "switchyard-core" version = "0.1.0" @@ -1770,7 +1732,6 @@ dependencies = [ "serde", "serde_json", "switchyard-components", - "switchyard-components-v2", "switchyard-core", "switchyard-libsy", "switchyard-protocol", @@ -2357,19 +2318,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" -[[package]] -name = "yaml_serde" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c7c1b1a6a7c8a6b2741a6c21a4f8918e51899b111cfa08d1288202656e3975" -dependencies = [ - "indexmap", - "itoa", - "libyaml-rs", - "ryu", - "serde", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 3e78719b..886688b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,6 @@ members = [ "crates/libsy", "crates/libsy-llm-client", "crates/switchyard-components", - "crates/switchyard-components-v2", - "crates/switchyard-components-v2-macros", "crates/switchyard-core", "crates/switchyard-py", "crates/protocol", diff --git a/README.md b/README.md index 40a38911..5e1b696d 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ write yourself. **Launcher routing is explicit.** By default, launchers use the built-in LLM-classifier router, which you tune with `--weak-model`, `--classifier-model`, `--profile`, and `--classifier-min-confidence`. Use `--model X` for -single-model passthrough. The `--routing-profiles FILE` path is deprecated and -remains only for launcher-owned legacy bundles. +single-model passthrough or `--routing-profiles FILE` for a YAML route bundle. ## Features @@ -72,12 +71,12 @@ switchyard launch openclaw --model openai/gpt-4o-mini --api-key "$OPENROUTER_API ``` Each launcher starts a local proxy, points the agent at it, and shuts the proxy -down when the agent exits. Use `--model` for single-model passthrough. The -deprecated `--routing-profiles` flag remains for launcher-owned legacy bundles: +down when the agent exits. Use `--model` for single-model passthrough or +`--routing-profiles` for a route bundle: ```bash switchyard launch claude --model openai/gpt-4o-mini --base-url https://openrouter.ai/api/v1 # single-model passthrough -switchyard --routing-profiles routes.yaml -- launch claude # legacy route bundle +switchyard --routing-profiles routes.yaml -- launch claude # route bundle ``` > **Bedrock-backed profile caveat (Claude Code + MCP):** Bedrock enforces a 64-character `toolSpec.name` cap. Claude Code's MCP bridge can auto-inject longer tool names, producing `BedrockException` 400s on tool-bearing requests. If you use a Bedrock-backed route and hit this, swap to an OpenAI-compatible model with `--model openai/gpt-4o` or a routing-profile YAML. @@ -86,58 +85,39 @@ See [Agent Launchers](docs/guides/agent_launchers.md) for supported harness versions, model requirements, troubleshooting, and Claude Code `/model` picker aliasing. -### 2. Run a standalone profile-config server +### 2. Run a standalone Python server -New standalone deployments use a profile config that separates provider -connectivity, upstream targets, and client-facing profiles. A complete -OpenRouter-backed random-routing config looks like this: +Define one or more client-facing routes in YAML. A complete OpenRouter-backed +random-routing bundle looks like this: ```yaml -endpoints: - openrouter: - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - -targets: - strong: - endpoint: openrouter - model: openai/gpt-4o - format: openai - weak: - endpoint: openrouter - model: openai/gpt-4o-mini - format: openai - -profiles: +defaults: + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + +routes: smart: - type: random-routing - strong: strong - weak: weak + type: random_routing + strong: + model: openai/gpt-4o + weak: + model: openai/gpt-4o-mini strong_probability: 0.3 + fallback_target_on_evict: strong ``` -Serve it as a proxy. The `smart` profile and both target ids are exposed as -models; clients select one through the request's `model` field: +Serve it as a proxy. The route name is exposed as a model ID, and clients +select it through the request's `model` field: ```bash -switchyard serve --config profiles.yaml --port 4000 +switchyard --routing-profiles routes.yaml -- serve --port 4000 curl http://localhost:4000/v1/models curl http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{"model": "smart", "messages": [{"role": "user", "content": "hi"}]}' ``` -> **Launcher compatibility:** Launcher subcommands do not accept `--config`. -> The deprecated `--routing-profiles` flag remains for launcher-owned legacy -> `routes:` bundles and saved bundle paths: - -```yaml -routes: - fast: - type: model - target: openai/gpt-4o-mini -``` - ```bash switchyard --routing-profiles routes.yaml -- launch claude switchyard --routing-profiles routes.yaml -- configure --target provider \ @@ -148,6 +128,10 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \ Non-interactive `configure` does not read provider credentials from the routing bundle; pass `--api-key` explicitly when persisting the bundle for CI. +The Rust `switchyard-server` binary has a separate explicit TOML schema for +LLM clients, targets, and libsy algorithms. See +[`crates/switchyard-server/README.md`](crates/switchyard-server/README.md). + For profile selection and full configuration examples, start with [Routing Overview](docs/routing_algorithms/overview.md), then open the strategy-specific page: diff --git a/crates/switchyard-components-v2-macros/Cargo.toml b/crates/switchyard-components-v2-macros/Cargo.toml deleted file mode 100644 index 0ea8c8e8..00000000 --- a/crates/switchyard-components-v2-macros/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "switchyard-components-v2-macros" -version = "0.1.0" -description = "Macros for experimental Switchyard components-v2 profile definitions" -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -[lib] -proc-macro = true - -[dependencies] -proc-macro2 = "1" -quote = "1" -syn = { version = "2", features = ["full"] } diff --git a/crates/switchyard-components-v2-macros/src/lib.rs b/crates/switchyard-components-v2-macros/src/lib.rs deleted file mode 100644 index 9be222d9..00000000 --- a/crates/switchyard-components-v2-macros/src/lib.rs +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Attribute macros for concise components-v2 profile config structs. - -use proc_macro::TokenStream; -use quote::{format_ident, quote}; -use syn::{ - parse_macro_input, Attribute, Field, Fields, GenericArgument, ItemStruct, LitStr, - PathArguments, Type, -}; - -/// Adds standard config derives, strict serde behavior, and a stable profile type. -#[proc_macro_attribute] -pub fn profile_config(args: TokenStream, input: TokenStream) -> TokenStream { - let kind = parse_macro_input!(args as LitStr); - let input = parse_macro_input!(input as ItemStruct); - emit_profile_config_struct(input, kind).into() -} - -// Re-emits the struct with Switchyard's standard profile-config surface. -fn emit_profile_config_struct(input: ItemStruct, kind: LitStr) -> proc_macro2::TokenStream { - let ident = input.ident.clone(); - let raw_ident = format_ident!("__{}Raw", ident); - let named_fields = match &input.fields { - Fields::Named(fields) => fields, - Fields::Unnamed(_) | Fields::Unit => { - return syn::Error::new_spanned( - input, - "profile_config only supports structs with named fields", - ) - .to_compile_error(); - } - }; - - let mut original_fields = Vec::new(); - let mut raw_fields = Vec::new(); - let mut build_fields = Vec::new(); - - for field in &named_fields.named { - let Some(field_ident) = field.ident.clone() else { - return syn::Error::new_spanned(field, "profile_config requires named fields") - .to_compile_error(); - }; - let has_profile_target = has_profile_target_attr(&field.attrs); - let cleaned_field = field_without_profile_target_attr(field); - let raw_ty = if has_profile_target { - match profile_target_raw_ty(&field.ty) { - Ok(raw_ty) => raw_ty, - Err(error) => return error.to_compile_error(), - } - } else { - field.ty.clone() - }; - let mut raw_field = cleaned_field.clone(); - raw_field.ty = raw_ty; - let value = if has_profile_target { - match profile_target_resolver(&field_ident, &field.ty) { - Ok(value) => value, - Err(error) => return error.to_compile_error(), - } - } else { - quote! { raw.#field_ident } - }; - - original_fields.push(cleaned_field); - raw_fields.push(raw_field); - build_fields.push(quote! { #field_ident: #value }); - } - - let mut original = input.clone(); - original.fields = Fields::Named(syn::FieldsNamed { - brace_token: named_fields.brace_token, - named: original_fields.into_iter().collect(), - }); - - quote! { - #[derive( - Clone, - Debug, - PartialEq, - ::serde::Serialize, - ::serde::Deserialize, - )] - #[serde(deny_unknown_fields)] - #original - - #[derive(::serde::Deserialize)] - #[serde(deny_unknown_fields)] - struct #raw_ident { - #(#raw_fields,)* - } - - impl #ident { - /// Stable discriminator used when this profile appears in serialized config. - pub const PROFILE_TYPE: &'static str = #kind; - - /// Returns the stable serialized discriminator for this profile config. - pub fn profile_type(&self) -> &'static str { - Self::PROFILE_TYPE - } - } - - impl ::switchyard_components_v2::__private::ProfileConfigDefinition for #ident { - const PROFILE_TYPE: &'static str = #kind; - - fn parse_profile_config( - value: ::serde_json::Value, - env: &::switchyard_components_v2::__private::ProfileBuildEnv<'_>, - ) -> ::switchyard_core::Result { - let raw: #raw_ident = ::serde_json::from_value(value).map_err(|error| { - ::switchyard_core::SwitchyardError::InvalidConfig(format!( - "failed to parse {} profile config: {error}", - #kind - )) - })?; - Ok(Self { - #(#build_fields,)* - }) - } - } - - const _: () = { - fn assert_profile_config() {} - let _ = assert_profile_config::<#ident>; - }; - } -} - -// Removes the helper attribute before the real struct reaches the compiler. -fn field_without_profile_target_attr(field: &Field) -> Field { - let mut cleaned = field.clone(); - cleaned.attrs = field - .attrs - .iter() - .filter(|attr| !is_profile_target_attr(attr)) - .cloned() - .collect(); - cleaned -} - -fn has_profile_target_attr(attrs: &[Attribute]) -> bool { - attrs.iter().any(is_profile_target_attr) -} - -fn is_profile_target_attr(attr: &Attribute) -> bool { - attr.path().is_ident("profile_target") -} - -fn profile_target_raw_ty(ty: &Type) -> syn::Result { - if is_type_ident(ty, "LlmTarget") { - Ok(syn::parse_quote! { ::switchyard_core::LlmTargetId }) - } else if let Some(inner) = single_generic_arg(ty, "Vec") { - if is_type_ident(inner, "LlmTarget") { - Ok(syn::parse_quote! { Vec<::switchyard_core::LlmTargetId> }) - } else { - Err(syn::Error::new_spanned( - ty, - "#[profile_target] Vec fields must contain LlmTarget", - )) - } - } else if let Some(inner) = single_generic_arg(ty, "Option") { - if is_type_ident(inner, "LlmTarget") { - Ok(syn::parse_quote! { Option<::switchyard_core::LlmTargetId> }) - } else { - Err(syn::Error::new_spanned( - ty, - "#[profile_target] Option fields must contain LlmTarget", - )) - } - } else { - Err(syn::Error::new_spanned( - ty, - "#[profile_target] supports LlmTarget, Vec, and Option", - )) - } -} - -fn profile_target_resolver( - field_ident: &syn::Ident, - ty: &Type, -) -> syn::Result { - if is_type_ident(ty, "LlmTarget") { - Ok(quote! { env.target(&raw.#field_ident)?.clone() }) - } else if single_generic_arg(ty, "Vec").is_some() { - Ok(quote! { - raw.#field_ident - .into_iter() - .map(|target_id| env.target(&target_id).map(Clone::clone)) - .collect::<::switchyard_core::Result>>()? - }) - } else if single_generic_arg(ty, "Option").is_some() { - Ok(quote! { - match raw.#field_ident { - Some(target_id) => Some(env.target(&target_id)?.clone()), - None => None, - } - }) - } else { - Err(syn::Error::new_spanned( - ty, - "#[profile_target] supports LlmTarget, Vec, and Option", - )) - } -} - -fn is_type_ident(ty: &Type, ident: &str) -> bool { - let Type::Path(type_path) = ty else { - return false; - }; - type_path.path.segments.last().is_some_and(|segment| { - segment.ident == ident && matches!(segment.arguments, PathArguments::None) - }) -} - -fn single_generic_arg<'a>(ty: &'a Type, outer: &str) -> Option<&'a Type> { - let Type::Path(type_path) = ty else { - return None; - }; - let segment = type_path.path.segments.last()?; - if segment.ident != outer { - return None; - } - let PathArguments::AngleBracketed(args) = &segment.arguments else { - return None; - }; - let mut args = args.args.iter(); - let Some(GenericArgument::Type(inner)) = args.next() else { - return None; - }; - if args.next().is_some() { - return None; - } - Some(inner) -} diff --git a/crates/switchyard-components-v2/Cargo.toml b/crates/switchyard-components-v2/Cargo.toml deleted file mode 100644 index 8e6b0637..00000000 --- a/crates/switchyard-components-v2/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "switchyard-components-v2" -version = "0.1.0" -description = "Experimental flatter Switchyard component profile runtime" -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -[dependencies] -async-stream = "0.3" -async-trait = "0.1" -futures-util = "0.3" -parking_lot = "0.12" -rand = "0.8" -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -switchyard-components = { path = "../switchyard-components" } -switchyard-components-v2-macros = { path = "../switchyard-components-v2-macros" } -switchyard-core = { path = "../switchyard-core" } -switchyard-protocol = { path = "../protocol" } -toml = "1.1" -tracing = { version = "0.1", default-features = false, features = ["std"] } -yaml_serde = "0.10" - -[dev-dependencies] -futures-core = "0.3" -tokio = { version = "1", features = ["macros", "rt", "sync"] } diff --git a/crates/switchyard-components-v2/DESIGN.md b/crates/switchyard-components-v2/DESIGN.md deleted file mode 100644 index 3da7e451..00000000 --- a/crates/switchyard-components-v2/DESIGN.md +++ /dev/null @@ -1,902 +0,0 @@ -# Switchyard Profile V2 Design - -Audience: contributors adding or reviewing Switchyard profiles in Rust, Python, or bindings. - -Prerequisites: familiarity with `ChatRequest`, `ChatResponse`, `LlmTarget`, async execution, and -the existing Switchyard request path. You do not need to know the older factory, bundle, graph, or -processor pipeline internals to understand v2. - -## Summary - -V2 is the simpler component model for Switchyard as a whole. The first implementation lives in -`switchyard-components-v2`, but the idea is not Rust-only: Python should converge on the same -profile-centered model instead of keeping a separate factory/bundle architecture. - -V2 replaces the older habit of decomposing every behavior into separate request processors, -backends, response processors, graph fragments, bundles, and factories with one central runtime -unit: - -```text -Profile -``` - -A profile owns the behavior for one addressable Switchyard mode. That can be passthrough, -random routing, latency-aware routing, a classifier stage-router, observability-only middleware, or a -future agent-facing policy. The profile decides how a request is prepared, which backend is called, -what response cleanup is needed, and where observability is recorded. - -The goal is not to make Switchyard less capable. The goal is to stop spreading one behavior across -many abstractions when most contributors need to reason about one thing: "what happens when this -profile receives a request?" - -## Why V2 Exists - -The original Switchyard shape was even looser than today's Python chain: a linked list of -"strategies" that accepted `Any` and returned `Any`. That made early experiments easy, but it also -made behavior hard to type, validate, compose, and review. The Python chain was the first cleanup: -split the untyped strategy chain into explicit request, backend, and response roles: - -```text -request-side work -> LLMBackend -> response-side work -``` - -That shape fixed real problems and made incremental porting possible, but it also carried Python -object boundaries into places where they no longer help. A single product feature could require -edits in several places: - -- request processor config, -- request processor implementation, -- backend config, -- backend implementation, -- response processor config, -- response processor implementation, -- graph or table wiring, -- factory or bundle glue, -- tests that needed to know the glue shape. - -For new contributors, that is the wrong entry point. They should not need to learn an orchestration -framework before implementing a routing policy, whether they are writing the implementation in Rust -or exposing it through Python. - -V2 starts from the product-level object instead: - -```text -one profile config -> one profile runtime -> one request lifecycle -``` - -This makes the code easier to read, easier to test, and harder to accidentally over-abstract. - -The config side follows the same principle. A config file should describe endpoints, targets, and -profiles. It should not ask users to assemble request processors, response processors, graph nodes, -bundles, or backend wrappers by hand. - -## Core Mental Model - -The v2 runtime has two surfaces: - -- an object-safe serving surface for code that only needs to run a profile; -- a typed hook surface for code that wants to inspect or embed the profile's request and response - hooks. - -In Rust, that target shape is: - -```rust -#[async_trait] -pub trait Profile: Send + Sync { - async fn run(&self, input: ProfileInput) -> Result; -} - -#[async_trait] -pub trait ProfileHooks: Send + Sync { - type ProcessedRequest: Send + Sync; - - async fn process(&self, input: ProfileInput) -> Result; - - async fn rprocess( - &self, - processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result; -} -``` - -Conceptually: - -```text -process(request) -> profile-specific processed request -run(request) -> final response -rprocess(processed, response) -> processed response -``` - -`Profile` is the erased serving contract. Servers, config loaders, and generic profile tables -can store `Box` and call `run()` without knowing a profile's private request -state. - -`ProfileHooks` is the typed authoring and embedding contract. The associated `ProcessedRequest` -type is the profile-owned wrapper around the prepared request and any request-side decision the -profile made. It replaces untyped metadata bags and avoids forcing every profile into a shared -decision enum. - -These methods are three related entry points into the same profile, not three required stages of a -hidden pipeline: - -- `run()` is the authoritative serving path. It owns the complete request lifecycle and is the - method a Switchyard server should call. -- `process()` is the request-side library hook. It returns the prepared request plus any typed - profile-specific state needed later, such as a random-routing decision or an initial - latency-service target selection. -- `rprocess()` is the response-side library hook. It receives the processed request state and the - backend response so the profile can perform cleanup, normalization, or accounting without a - side-channel. - -`run()` should normally call `process()` and `rprocess()` when those hooks express the same -lifecycle cleanly. If `process()` seems unable to carry the state that `run()` needs, the first fix -is to make that profile's `ProcessedRequest` type richer, not to add a generic metadata map. A -profile should bypass its hooks only when the lifecycle genuinely cannot be represented by a -request-side wrapper, and that reason should be documented on the concrete profile. - -The self-contained serving flow looks like this: - -```text -HTTP / SDK wire request - | - | endpoint + translation layer - v -ChatRequest - | - | Profile::run() - | - process into profile-specific request state - | - prepare or rewrite the backend request - | - choose target/backend - | - call backend - | - normalize response - | - record stats/errors - v -ChatResponse - | - | endpoint + translation layer - v -HTTP / SDK wire response -``` - -`process()` and `rprocess()` are intentionally still present. They are not a return to the old -pipeline hierarchy. They exist because Switchyard can be embedded as middleware in systems that -own their own transport or model runtime: - -```text -caller-owned request object - | - | caller converts or already has a ChatRequest - | - v -ChatRequest - | - | Profile::process() - v -profile-specific processed request - | - | processed.request() - v -prepared ChatRequest - | - | caller-owned HTTP client, model runner, batcher, or agent runtime - v -ChatResponse - | - | Profile::rprocess(processed, response) - v -processed ChatResponse - | - | caller returns or translates response - v -caller-owned response object -``` - -In middleware mode, the reusable pieces are the profile's request hook, response hook, shared -backend adapters, translation layer, and explicit helpers such as stats recorders. The caller owns -transport and the actual model call. The profile still owns the request and response behavior it -exposes through those hooks. - -The important rule is that `process()` must return more than a bare `ChatRequest` when the profile -makes a meaningful request-side decision. For example, random routing should return a -`RandomRoutingProcessedRequest` containing both the rewritten request and the routing decision. -Latency-service routing can return a `LatencyServiceProcessedRequest` containing the rewritten -request and the selected target candidate. Those wrappers are profile-specific types, not variants -of one universal metadata object. - -Do not implement unsupported hooks with `panic!`, `unimplemented!()`, or runtime -`NotImplementedError`. If every profile must expose the hook, make the hook total with a meaningful -identity wrapper for simple profiles. If a future capability only applies to some profiles, split it -into a separate trait instead of adding a method that most profiles cannot honestly implement. - -## What A Profile Owns - -A profile owns the policy boundary. If a reviewer asks, "where is this behavior implemented?", the -answer should usually be one module under `src/profiles/`. - -For example: - -```text -src/profiles/passthrough.rs -src/profiles/random_routing.rs -src/profiles/latency_service/ -``` - -Each profile should keep these pieces close together: - -- the user-facing config struct, -- validation that is specific to that config, -- runtime construction, -- request-side behavior, -- the processed-request wrapper returned by `process()`, -- any profile-specific request decision type, -- backend selection and calls, -- response-side behavior, -- stats and error accounting, -- focused tests for the profile's invariants. - -Shared engines are still allowed. Random routing can reuse `RandomRoutingEngine`. Latency routing -can split health polling and selection into small private modules. The distinction is that these -helpers are implementation details of a profile, not a second public orchestration model. - -## What V2 Removes From The Contributor Path - -V2 is designed so profile authors do not have to build or understand these concepts: - -- `ProxyContext` as a mutable cross-stage side channel, -- request processor trait objects, -- response processor trait objects, -- graph fragments as a profile output format, -- table builders as the primary authoring API, -- middleware bundles, -- factories, -- served-model side channels. - -Those ideas may still exist elsewhere during migration or for compatibility. They should not be -the design vocabulary for new profile code. - -For now, v2 does not treat backward compatibility as a requirement. If an older Python factory, -bundle, route shape, or wrapper only exists to preserve the previous object hierarchy, it is a -candidate for removal rather than preservation. Temporary aliases can be added later only when a -release decision explicitly needs them; they should not shape the core design. - -## Deliberate Helpers Instead Of Implicit Middleware - -Helpers are still valuable. The difference is that v2 makes helper use visible at the profile -boundary. - -Stats are the clearest example. In the older model, stats could be hidden behind separate request -processors, response processors, or a stats backend wrapper. That hid the control flow: - -```text -request processor records start -backend records selection -response processor records usage -``` - -The profile author then had to know which side channel connected those pieces. That side channel -was often `ProxyContext`. - -In v2, a profile records stats at the semantic point where it knows the truth: - -```rust -let backend_started_at = Instant::now(); -let response = match selected_backend.call(processed.request().clone()).await { - Ok(response) => response, - Err(error) => { - self.record_error(&processed)?; - return Err(error); - } -}; - -let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; -self.record_success(&processed, Some(backend_latency_ms))?; - -let usage = response.body().map(usage_from_body).unwrap_or_default(); -self.record_usage(&processed, usage, Some(total_latency_ms), Some(routing_overhead_ms))?; -``` - -This is more explicit, and that is the point. The profile knows: - -- which target was selected, -- which model should receive accounting, -- which profile-specific decision should receive accounting, -- which failures happened before a response existed, -- which latency is backend time versus profile overhead. - -A stats helper should reduce mechanical work, not hide the lifecycle. A good helper answers, -"how do I record this known event consistently?" A bad helper answers, "where did this event come -from?" by forcing the reader to chase implicit middleware. - -## Backend Reuse - -V2 does not require rewriting every backend immediately. The crate currently adapts existing native -OpenAI and Anthropic backends through a small `ProfileBackend` surface that does not expose -`ProxyContext`. - -That adapter is a migration tool. The target direction is: - -```text -Profile - owns routing and lifecycle -Backend - owns provider call behavior -Translation - owns wire-format conversion -``` - -Backends remain a reasonable abstraction because provider calls are real reusable behavior. The -problem v2 fixes is not "all abstractions are bad." The problem is splitting one profile's policy -across too many orchestration objects. - -## Config Shape - -Each profile config lives beside its runtime and declares a stable serialized type. In Rust, the -macro handles the repetitive serde and config-resolution surface: - -```rust -#[profile_config("random-routing")] -pub struct RandomRoutingProfileConfig { - #[profile_target] - pub strong: LlmTarget, - #[profile_target] - pub weak: LlmTarget, - #[serde(default = "default_strong_probability")] - pub strong_probability: f64, - #[serde(default)] - pub rng_seed: Option, -} -``` - -The macro standardizes the boring parts: - -- clone/debug/serde derives, -- strict unknown-field rejection, -- a stable `PROFILE_TYPE`, -- `profile_type()`, -- profile-owned parsing from serialized config, -- `#[profile_target]` resolution from config IDs into concrete `LlmTarget` values. - -It should not hide profile-specific meaning. Cross-field validation belongs in normal Rust when it -is policy-specific. For example, validating duplicate latency targets is easier to review as plain -code than as macro magic. - -Python should follow the same rule even if the mechanism is a decorator or class registration -instead of a Rust proc macro. A profile's config belongs next to the profile implementation; it -should not live in a distant central factory file. - -The intended config authoring rule is: - -```text -shared config behavior can be macro-generated -profile policy stays in the profile module -``` - -## User-Facing Config Model - -The v2 config loader is the user-facing entry point for profile construction. It accepts the same -schema as YAML, JSON, or TOML, with `${VAR}` interpolation before typed deserialization: - -```yaml -endpoints: - nvidia: - base_url: https://integrate.api.nvidia.com/v1 - api_key: ${NVIDIA_API_KEY} - timeout_secs: 120.0 - -# Targets are named by the model id they serve. -targets: - nvidia/frontier-model: - endpoint: nvidia - model: nvidia/frontier-model - format: openai - - nvidia/fast-model: - endpoint: nvidia - model: nvidia/fast-model - format: openai - -profiles: - direct: - type: passthrough - target: nvidia/fast-model - - smart-stage-router: - type: stage_router - capable: nvidia/frontier-model - efficient: nvidia/fast-model - fallback_target_on_evict: nvidia/frontier-model - picker: capable_first - confidence_threshold: 0.7 -``` - -The intended load path is: - -```text -YAML / JSON / TOML - | - v -ProfileConfigDocument - | - v -ProfileConfigPlan - | - v -Box -``` - -`ProfileConfigDocument` is the parsed file shape. It is allowed to contain config-facing -references, such as `target: nvidia/fast-model` or `endpoint: nvidia`. It is not runtime-ready. - -`ProfileConfigPlan` is the resolved plan. It has concrete `LlmTarget` values and typed profile -configs. The design deliberately does not rebuild the old graph/table/factory system. The plan is -only the point where a parsed config has been validated enough to build one profile or all profiles. - -The `ProfileConfig` trait is the build contract that keeps runtime construction explicit. The macro -can generate parsing and target-resolution glue, but every profile config still has to say how it -turns into its runtime profile. The concrete runtime must implement the object-safe `Profile` -runtime contract and the typed `ProfileHooks` embedding contract; config plans erase only to -`Profile` for serving. - -The config split should be enforced with a trait shaped like this: - -```rust -pub trait ProfileConfig: ProfileConfigDefinition { - type Runtime: Profile + ProfileHooks + 'static; - - fn build(&self) -> Result; - - fn build_boxed(&self) -> Result> { - Ok(Box::new(self.build()?)) - } -} -``` - -That means a profile config can rely on macro-generated parsing, but it cannot skip runtime -construction. The `#[profile_config]` macro emits a compile-time assertion that the config -implements `ProfileConfig`, so the compiler requires each config to provide `build()`. Because -`type Runtime: Profile + ProfileHooks`, a profile that only implements `run()` but leaves -`process()` or `rprocess()` out does not satisfy the design; it should provide real hook behavior -or the hook should move to a narrower capability trait. - -```rust -impl ProfileConfig for RandomRoutingProfileConfig { - type Runtime = RandomRoutingProfile; - - fn build(&self) -> Result { - let router = RandomRoutingEngine::new( - RandomRoutingProcessorConfig::new(self.strong.clone(), self.weak.clone()) - .with_strong_probability(self.strong_probability)? - .with_rng_seed(self.rng_seed), - )?; - - Ok(RandomRoutingProfile { - router, - strong_backend: native_target_backend(self.strong.clone())?, - weak_backend: native_target_backend(self.weak.clone())?, - stats: profile_stats_accumulator(), - }) - } -} -``` - -This is deliberate. The macro handles the repetitive config surface. The profile author still owns -the runtime decisions: validation, backend construction, poller construction, stats handles, and -any profile-specific resources. - -The central config loader should own only generic work: - -- format detection and loading, -- YAML/JSON/TOML parsing, -- environment interpolation, -- shared endpoint inheritance, -- target resolution, -- dispatch by profile `type`, -- building requested profile runtimes. - -Profile modules own everything profile-specific: - -- which config fields exist, -- which fields are target references, -- cross-field validation, -- backend/runtime construction, -- profile-specific error semantics. - -This is why `#[profile_target]` matters. The file can say `strong: strong`, but the runtime config -still receives a real `LlmTarget`. The profile author writes the natural runtime type, while the -macro handles the file-facing ID indirection. - -The generated `profile_types!` registry is the narrow central list of supported profile config -types. It is acceptable because it is one line per profile and only dispatches to profile-owned -parsing/building. It should not grow into a central hand-written schema with duplicated fields. - -## Complete Profile Shape - -The Rust shape should make the full lifecycle visible in one profile module: - -```rust -#[profile_config("random-routing")] -pub struct RandomRoutingProfileConfig { - #[profile_target] - pub strong: LlmTarget, - #[profile_target] - pub weak: LlmTarget, - pub strong_probability: f64, - pub rng_seed: Option, -} - -impl ProfileConfig for RandomRoutingProfileConfig { - type Runtime = RandomRoutingProfile; - - fn build(&self) -> Result { - validate_probability(self.strong_probability)?; - - Ok(RandomRoutingProfile { - router: RandomRoutingEngine::from_config(self)?, - strong_backend: native_target_backend(self.strong.clone())?, - weak_backend: native_target_backend(self.weak.clone())?, - stats: profile_stats_accumulator(), - }) - } -} - -pub struct RandomRoutingProfile { - router: RandomRoutingEngine, - strong_backend: TargetBackend, - weak_backend: TargetBackend, - stats: StatsAccumulator, -} - -pub struct RandomRoutingProcessedRequest { - request: ChatRequest, - decision: RandomRoutingDecision, -} - -impl ProfileProcessedRequest for RandomRoutingProcessedRequest { - fn request(&self) -> &ChatRequest { - &self.request - } -} - -#[async_trait] -impl ProfileHooks for RandomRoutingProfile { - type ProcessedRequest = RandomRoutingProcessedRequest; - - async fn process(&self, request: ChatRequest) -> Result { - let (request, decision) = self.route_request(request)?; - Ok(RandomRoutingProcessedRequest { request, decision }) - } - - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for RandomRoutingProfile { - async fn run(&self, request: ChatRequest) -> Result { - let started_at = Instant::now(); - let processed = self.process(request).await?; - let backend = self.backend_for(&processed.decision)?; - - let response = match backend.call(processed.request().clone()).await { - Ok(response) => response, - Err(error) => { - self.record_error(&processed.decision)?; - return Err(error); - } - }; - - self.record_success_and_usage(&processed.decision, &response, started_at)?; - self.rprocess(&processed, response).await - } -} -``` - -The equivalent Python shape should follow the same object boundary. Python may use decorators -instead of Rust macros, but it should not return to factory dictionaries or hidden bundles: - -```python -ProcessedRequestT = TypeVar("ProcessedRequestT") - - -class Profile(Protocol): - async def run(self, request: ChatRequest) -> ChatResponse: ... - - -class ProfileHooks(Protocol[ProcessedRequestT]): - async def process(self, request: ChatRequest) -> ProcessedRequestT: ... - - async def rprocess( - self, - processed: ProcessedRequestT, - response: ChatResponse, - ) -> ChatResponse: ... - - -@profile_config("random-routing") -@dataclass -class RandomRoutingProfileConfig: - strong: LlmTarget - weak: LlmTarget - strong_probability: float = 0.5 - rng_seed: int | None = None - - def build(self) -> "RandomRoutingProfile": - validate_probability(self.strong_probability) - return RandomRoutingProfile( - router=RandomRoutingEngine( - strong=self.strong, - weak=self.weak, - strong_probability=self.strong_probability, - rng_seed=self.rng_seed, - ), - strong_backend=native_target_backend(self.strong), - weak_backend=native_target_backend(self.weak), - stats=profile_stats_accumulator(), - ) - - -@dataclass -class RandomRoutingProcessedRequest: - request: ChatRequest - decision: RandomRoutingDecision - - -class RandomRoutingProfile(Profile[RandomRoutingProcessedRequest]): - def __init__( - self, - *, - router: RandomRoutingEngine, - strong_backend: TargetBackend, - weak_backend: TargetBackend, - stats: StatsAccumulator, - ) -> None: - self.router = router - self.strong_backend = strong_backend - self.weak_backend = weak_backend - self.stats = stats - - async def process(self, request: ChatRequest) -> RandomRoutingProcessedRequest: - request, decision = self._route_request(request) - return RandomRoutingProcessedRequest(request=request, decision=decision) - - async def run(self, request: ChatRequest) -> ChatResponse: - started_at = monotonic() - processed = await self.process(request) - backend = self._backend_for(processed.decision) - - try: - response = await backend.call(processed.request) - except SwitchyardError: - self._record_error(processed.decision) - raise - - self._record_success_and_usage(processed.decision, response, started_at) - return await self.rprocess(processed, response) - - async def rprocess( - self, - processed: "RandomRoutingProcessedRequest", - response: ChatResponse, - ) -> ChatResponse: - return response -``` - -The language-specific syntax can differ. The architecture should not. A profile config builds one -profile runtime, and the profile runtime owns the visible request lifecycle. - -## How A Request Flows In Existing Profiles - -### Passthrough - -Passthrough has no routing state, so its `run()` can be the straightforward lifecycle: - -```text -process(request) - | - v -PassthroughProcessedRequest { request } - | - v -single target backend call - | - v -record success, error, latency, and usage - | - v -rprocess(processed, response) -``` - -This is the simplest profile and should remain the reference for a one-target policy. - -### Random Routing - -Random routing selects between configured targets. Its request-side decision belongs in its own -processed-request type: - -```text -process(request) - | - +--> rewritten request model - | - +--> RandomRoutingDecision - | - v - selected backend call - | - v - stats for selected decision -``` - -The key is that `RandomRoutingProcessedRequest` is not a generic metadata wrapper. It is the -random-routing profile's own type. If another profile makes a different kind of request-side -decision, it gets a different processed-request type. - -### Latency Service - -Latency-service routing owns a health cache and chooses among multiple targets: - -```text -health cache - | - v -process(request) selects initial target candidate - | - v -call selected backend - | - +--> success: record usage and return - | - +--> failure: record error, exclude target, retry another target -``` - -The hot path reads cached health. Health polling is explicit through `poll_once()` or external -health injection, which keeps serving free of accidental latency-service network calls. -`LatencyServiceProcessedRequest` may contain the prepared request and the initial target selection, -but the full retry loop still belongs in `run()` because retries update per-call exclusion state as -backend calls fail. - -## Why This Helps Contributions - -V2 makes the contribution unit match the feature unit. A contributor adding a routing policy should -be able to open one profile module and answer: - -- What does the config look like? -- What gets validated? -- How does request mutation work? -- Which backend is called? -- What happens on backend error? -- What stats are recorded? -- Can this profile be used as middleware-only? -- What tests define the behavior? - -That is a smaller review surface than a graph of processors, bundles, factories, and table entries. - -It also makes tests more direct. Instead of testing whether factory glue assembled the right -sequence of objects, tests can call: - -```rust -let processed = profile.process(request).await?; -profile.run(request).await?; -profile.rprocess(&processed, response).await?; -``` - -This encourages adversarial tests around actual behavior: - -- invalid config is rejected, -- request models are rewritten correctly, -- selected target state does not leak across concurrent calls, -- backend failures record stats and propagate errors, -- retries do not hit the same failed target when alternatives remain, -- middleware-only hooks do not accidentally call a backend. - -## Adding A New Profile - -Start with the smallest profile-owned implementation. - -1. Create `src/profiles/.rs` or `src/profiles//mod.rs`. -2. Define `ProfileConfig` with `#[profile_config("")]`. -3. Mark config fields that refer to targets with `#[profile_target]`. -4. Implement the `ProfileConfig` build contract for the config type. -5. Put profile-specific validation near `build()`. -6. Build the runtime from fully resolved targets and existing backend helpers. -7. Define `ProcessedRequest` for the output of `process()`. -8. Include the prepared `ChatRequest` and any profile-specific request-side decision in that type. -9. Implement `ProfileHooks` and `Profile` for `Profile`. -10. Keep retry loops, backend calls, and mutable per-call state local to `run()`. -11. Add tests for config validation, hook behavior, full `run()` behavior, stats, errors, and - concurrency if the profile has mutable state. -12. Add the config type to the generated profile type list. -13. Export the config and runtime from `src/profiles/mod.rs` and `src/lib.rs`. - -For Python-defined profiles, the same shape applies: - -1. Put the profile config and runtime in a profile module, not in a global factory directory. -2. Use the shared profile registration path so config loading can discover it. -3. Keep validation and target resolution close to the profile. -4. Define a typed processed-request object for the profile's `process()` output. -5. Implement the same `process`, `run`, and `rprocess(processed, response)` lifecycle. -6. Make config fields explicit and typed rather than accepting arbitrary factory dictionaries. -7. Delete old factory or bundle glue instead of wrapping it indefinitely. - -The first version should be boring. Add private helpers only when they make the profile easier to -read. Do not add a new public abstraction because two profiles happen to share five lines. - -## What Not To Add Back - -Avoid reintroducing old shapes under new names: - -- A "profile graph" that must be built before a profile can run. -- A "profile bundle" that only wraps request and response hooks. -- A hidden stats layer that records without being visible in the profile's `run()`. -- A context object whose main job is carrying state between self-owned stages. -- A generic "profile metadata" object used as a new untyped side channel. -- A central factory registry that every new profile has to touch manually. - -If a new helper is needed, prefer one of these forms: - -- a private function inside one profile module, -- a small shared function with no lifecycle ownership, -- a reusable engine that performs pure policy logic, -- a backend adapter that only owns provider call mechanics. - -## Migration Boundary - -`switchyard-components-v2` is intentionally separate from the older -`switchyard-components` crate while the shape settles. During migration, v2 can reuse proven -engines and backend code from the older crate, but new profile code should not adopt the older -public shape. - -The same applies to Python. The Python package can keep working while profile v2 lands, but the -target state is not "Rust has profiles while Python keeps factories." The target state is one -profile model, with Python using bindings or Python-defined profiles that obey the same lifecycle. - -The practical migration stance is: - -```text -preserve behavior -do not preserve unnecessary object structure -do not preserve backward compatibility by default -``` - -This means a v2 profile can reuse a mature random-routing engine, stats accumulator, or native -backend implementation while still presenting a flatter runtime surface. - -## Staleness Risks - -This document will need updates when: - -- the temporary `ProfileBackend` adapter is replaced by native v2 backend ownership, -- config loading for v2 profiles becomes the primary server path, -- Python bindings expose v2 profiles directly or Python-defined profiles adopt the same lifecycle, -- stats recording moves from explicit profile calls to a deliberate helper with an equally visible - lifecycle, -- new profiles introduce a legitimate pattern that this document does not cover. - -## PyO3 Binding Caveat - -The associated `ProcessedRequest` type is deliberately good Rust, but it is not directly -object-safe across PyO3. Generic Python bindings should therefore split the surface the same way the -Rust traits do. - -The serving path is easy to expose generically: Python can hold a `Profile` wrapper backed by -`Arc` and call `run(request)` for any Rust profile returned by config loading. -That is the right default for servers, launchers, and generic profile tables. - -The hook path needs one more layer. Python cannot call a fully generic `dyn ProfileHooks` without -erasing the associated processed-request type. Bindings should expose either: - -- an opaque `ProcessedRequest` Python handle backed by an erased Rust wrapper, with a `.request` - accessor for caller-owned model calls, or -- macro-generated concrete Python classes for each registered profile and its concrete - `ProcessedRequest` type. - -Both approaches keep the core invariant: the processed-request value is typed and profile-owned in -Rust. Python should not receive a generic metadata dictionary, and `switchyard-components-v2` -should not depend on PyO3. PyO3 registration belongs in `switchyard-py`, ideally generated from the -same profile registry that lists the supported profile config types. diff --git a/crates/switchyard-components-v2/src/backend.rs b/crates/switchyard-components-v2/src/backend.rs deleted file mode 100644 index e61224d0..00000000 --- a/crates/switchyard-components-v2/src/backend.rs +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Backend helpers for profile-owned runtimes. - -use std::sync::Arc; - -use async_trait::async_trait; -use switchyard_components::backends::{AnthropicNativeBackend, OpenAiNativeBackend}; -use switchyard_core::{ - BackendFormat, ChatRequest, ChatResponse, LlmTarget, Result, SwitchyardError, -}; - -/// Backend trait used by v2 profiles without exposing `ProxyContext`. -#[async_trait] -pub trait ProfileBackend: Send + Sync { - /// Calls a backend with an already-prepared request. - async fn call(&self, request: &ChatRequest) -> Result; -} - -/// One target and the backend that serves it. -#[derive(Clone)] -pub struct TargetBackend { - target: LlmTarget, - backend: Arc, -} - -impl TargetBackend { - /// Creates a target/backend pair. - pub fn new(target: LlmTarget, backend: Arc) -> Self { - Self { target, backend } - } - - /// Returns the target metadata. - pub fn target(&self) -> &LlmTarget { - &self.target - } - - /// Calls the target backend with a request prepared by the profile. - pub async fn call(&self, request: &ChatRequest) -> Result { - self.backend.call(request).await - } -} - -/// Builds the existing native backend for one fully-resolved target. -pub(crate) fn native_target_backend(target: LlmTarget) -> Result { - let backend: Arc = match target.format { - BackendFormat::OpenAi | BackendFormat::Responses => { - Arc::new(OpenAiNativeBackend::new(target.clone())?) - } - BackendFormat::Anthropic => Arc::new(AnthropicNativeBackend::new(target.clone())?), - BackendFormat::Auto => { - return Err(SwitchyardError::InvalidConfig(format!( - "target {} must have a resolved backend format", - target.id - ))); - } - }; - Ok(TargetBackend::new(target, backend)) -} - -#[async_trait] -impl ProfileBackend for OpenAiNativeBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.call_without_context(request).await - } -} - -#[async_trait] -impl ProfileBackend for AnthropicNativeBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.call_without_context(request).await - } -} diff --git a/crates/switchyard-components-v2/src/config.rs b/crates/switchyard-components-v2/src/config.rs deleted file mode 100644 index 95e60cca..00000000 --- a/crates/switchyard-components-v2/src/config.rs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Profile config entry point split by lifecycle stage. -//! -//! Parsing produces a file-facing `ProfileConfigDocument`. Resolving turns that -//! document into an opaque `ProfileConfigPlan` with runtime-ready targets and -//! validated profile bodies. - -mod loading; -mod parsing; -mod resolving; - -pub use loading::{parse_profile_config_path, ProfileConfigFormat}; -pub use parsing::{ - parse_profile_config_str, parse_profile_config_str_with_env_lookup, ProfileConfigDocument, -}; -pub use resolving::{ProfileBuildEnv, ProfileConfigDefinition}; -pub use resolving::{ProfileConfig, ProfileConfigPlan}; diff --git a/crates/switchyard-components-v2/src/config/loading.rs b/crates/switchyard-components-v2/src/config/loading.rs deleted file mode 100644 index a36aaf55..00000000 --- a/crates/switchyard-components-v2/src/config/loading.rs +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Filesystem loading and format detection for profile configs. - -use std::fs; -use std::path::Path; - -use switchyard_core::{Result, SwitchyardError}; - -use super::parsing::ProfileConfigDocument; - -/// File format accepted by the v2 profile config loader. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ProfileConfigFormat { - /// JSON configuration. - Json, - /// TOML configuration. - Toml, - /// YAML configuration. - Yaml, -} - -impl ProfileConfigFormat { - /// Infers the config format from a path extension. - pub fn from_path(path: impl AsRef) -> Result { - let extension = path - .as_ref() - .extension() - .and_then(|extension| extension.to_str()) - .ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "profile config path {} has no file extension", - path.as_ref().display() - )) - })?; - match extension { - "json" => Ok(Self::Json), - "toml" => Ok(Self::Toml), - "yaml" | "yml" => Ok(Self::Yaml), - other => Err(SwitchyardError::InvalidConfig(format!( - "unsupported profile config extension `{other}`" - ))), - } - } -} - -impl ProfileConfigDocument { - /// Reads a profile config path and infers the format from its extension. - pub fn from_path(path: impl AsRef) -> Result { - let path = path.as_ref(); - let input = fs::read_to_string(path).map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "failed to read profile config {}: {error}", - path.display() - )) - })?; - Self::from_str(&input, ProfileConfigFormat::from_path(path)?) - } -} - -/// Reads and parses a profile config path, inferring the format from extension. -pub fn parse_profile_config_path(path: impl AsRef) -> Result { - ProfileConfigDocument::from_path(path) -} diff --git a/crates/switchyard-components-v2/src/config/parsing.rs b/crates/switchyard-components-v2/src/config/parsing.rs deleted file mode 100644 index e20fbb42..00000000 --- a/crates/switchyard-components-v2/src/config/parsing.rs +++ /dev/null @@ -1,312 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! String parsing and file-facing schema for profile configs. - -use std::collections::{BTreeMap, BTreeSet}; - -use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer}; -use serde_json::Value; -use switchyard_core::{ - BackendFormat, EndpointConfig, EndpointId, LlmTargetId, ModelId, ProfileId, Result, - SwitchyardError, -}; - -use super::loading::ProfileConfigFormat; - -/// User-facing profile config document produced by parsing. -/// -/// This is intentionally not runtime-ready yet. Endpoint inheritance, target -/// references, and profile-owned validation happen in the resolving stage. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ProfileConfigDocument { - /// Shared endpoint definitions referenced by targets. - #[serde(default)] - pub(super) endpoints: BTreeMap, - /// Concrete targets keyed by their stable target IDs. - #[serde(default)] - pub(super) targets: BTreeMap, - /// Named serialized profile bodies keyed by the model/profile ID exposed by serving. - #[serde(default)] - pub(super) profiles: BTreeMap, -} - -impl ProfileConfigDocument { - /// Parses a profile config string and interpolates `${VAR}` environment values. - pub fn from_str(input: &str, format: ProfileConfigFormat) -> Result { - parse_profile_config_str_with_env_lookup(input, format, |name| std::env::var(name).ok()) - } - - /// Iterates parsed profile IDs before profile-owned validation. - pub fn profile_ids(&self) -> impl Iterator { - self.profiles.keys() - } - - /// Returns the serialized type discriminator for one parsed profile. - pub fn profile_type(&self, profile_id: &ProfileId) -> Option<&str> { - self.profiles - .get(profile_id) - .map(SerializedProfileConfig::profile_type) - } - - /// Returns the profile-owned config body without the `type` discriminator. - pub fn profile_body(&self, profile_id: &ProfileId) -> Option<&Value> { - self.profiles - .get(profile_id) - .map(SerializedProfileConfig::body) - } - - /// Returns a profile's envelope `subagent_target` reference, if configured. - pub fn profile_subagent_target(&self, profile_id: &ProfileId) -> Option<&LlmTargetId> { - self.profiles - .get(profile_id) - .and_then(SerializedProfileConfig::subagent_target) - } - - /// Returns a copy of this document with the selected profiles removed. - pub fn without_profiles(&self, profile_ids: &[ProfileId]) -> Self { - let omitted = profile_ids.iter().collect::>(); - let mut document = self.clone(); - document - .profiles - .retain(|profile_id, _profile| !omitted.contains(profile_id)); - document - } -} - -/// Serialized profile body split into a type discriminator, common envelope -/// fields, and profile-owned fields. -#[derive(Clone, Debug, PartialEq)] -pub(super) struct SerializedProfileConfig { - profile_type: String, - subagent_target: Option, - body: Value, -} - -impl SerializedProfileConfig { - /// Creates a serialized profile config after verifying the body is an object. - fn new( - profile_type: impl Into, - subagent_target: Option, - body: Value, - ) -> Result { - let profile_type = profile_type.into(); - if profile_type.trim().is_empty() { - return Err(SwitchyardError::InvalidConfig( - "profile config `type` must not be empty".to_string(), - )); - } - if !body.is_object() { - return Err(SwitchyardError::InvalidConfig( - "profile config body must be an object".to_string(), - )); - } - Ok(Self { - profile_type, - subagent_target, - body, - }) - } - - /// Returns the profile type discriminator from the file's `type` field. - pub(super) fn profile_type(&self) -> &str { - &self.profile_type - } - - /// Returns the envelope `subagent_target` reference from the file, if any. - pub(super) fn subagent_target(&self) -> Option<&LlmTargetId> { - self.subagent_target.as_ref() - } - - /// Returns the profile-owned config fields without the envelope fields. - pub(super) fn body(&self) -> &Value { - &self.body - } -} - -impl<'de> Deserialize<'de> for SerializedProfileConfig { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let mut value = Value::deserialize(deserializer)?; - let Value::Object(map) = &mut value else { - return Err(de::Error::custom("profile config must be an object")); - }; - let profile_type = map - .remove("type") - .ok_or_else(|| de::Error::custom("profile config requires a `type` field"))?; - let Value::String(profile_type) = profile_type else { - return Err(de::Error::custom("profile config `type` must be a string")); - }; - // `subagent_target` is a common envelope field consumed here, like `type`, - // so profile-owned bodies (which deny unknown fields) never see it. - let subagent_target = match map.remove("subagent_target") { - None => None, - Some(Value::String(target_id)) => { - Some(LlmTargetId::new(target_id).map_err(de::Error::custom)?) - } - Some(_) => { - return Err(de::Error::custom( - "profile config `subagent_target` must be a target id string", - )); - } - }; - SerializedProfileConfig::new(profile_type, subagent_target, value) - .map_err(de::Error::custom) - } -} - -impl Serialize for SerializedProfileConfig { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - let Value::Object(mut map) = self.body.clone() else { - return Err(ser::Error::custom("profile config body must be an object")); - }; - map.insert("type".to_string(), Value::String(self.profile_type.clone())); - if let Some(subagent_target) = &self.subagent_target { - map.insert( - "subagent_target".to_string(), - Value::String(subagent_target.as_str().to_owned()), - ); - } - Value::Object(map).serialize(serializer) - } -} - -/// File-facing target config that can inherit from a shared endpoint. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TargetConfig { - /// Optional shared endpoint ID to inherit connection settings from. - #[serde(default)] - pub endpoint: Option, - /// Upstream model name sent to the provider. - pub model: ModelId, - /// Wire format expected by the upstream target. - pub format: BackendFormat, - /// Target-local base URL override. - #[serde(default)] - pub base_url: Option, - /// Target-local API key override. - #[serde(default)] - pub api_key: Option, - /// Target-local timeout override in seconds. - #[serde(default)] - pub timeout_secs: Option, - /// Per-target outbound request body extensions. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub extra_body: Option, - /// Per-target outbound request header extensions. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub extra_headers: BTreeMap, -} - -/// Parses a profile config string with the requested format. -pub fn parse_profile_config_str( - input: &str, - format: ProfileConfigFormat, -) -> Result { - ProfileConfigDocument::from_str(input, format) -} - -/// Parses a profile config string using a caller-provided environment lookup. -/// -/// This is useful for embedders and tests that need deterministic interpolation -/// without mutating process-global environment variables. -pub fn parse_profile_config_str_with_env_lookup( - input: &str, - format: ProfileConfigFormat, - lookup: impl Fn(&str) -> Option, -) -> Result { - parse_profile_config_str_with_lookup(input, format, lookup) -} - -fn parse_profile_config_str_with_lookup( - input: &str, - format: ProfileConfigFormat, - lookup: impl Fn(&str) -> Option, -) -> Result { - let mut value = parse_value(input, format)?; - interpolate_value(&mut value, &lookup)?; - serde_json::from_value(value).map_err(|error| { - SwitchyardError::InvalidConfig(format!("failed to parse profile config: {error}")) - }) -} - -fn parse_value(input: &str, format: ProfileConfigFormat) -> Result { - match format { - ProfileConfigFormat::Json => serde_json::from_str(input).map_err(|error| { - SwitchyardError::InvalidConfig(format!("failed to parse JSON profile config: {error}")) - }), - ProfileConfigFormat::Toml => { - let value = toml::from_str::(input).map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "failed to parse TOML profile config: {error}" - )) - })?; - serde_json::to_value(value).map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "failed to normalize TOML profile config: {error}" - )) - }) - } - ProfileConfigFormat::Yaml => yaml_serde::from_str(input).map_err(|error| { - SwitchyardError::InvalidConfig(format!("failed to parse YAML profile config: {error}")) - }), - } -} - -fn interpolate_value(value: &mut Value, lookup: &impl Fn(&str) -> Option) -> Result<()> { - match value { - Value::String(text) => { - *text = interpolate_string(text, lookup)?; - } - Value::Array(items) => { - for item in items { - interpolate_value(item, lookup)?; - } - } - Value::Object(map) => { - for item in map.values_mut() { - interpolate_value(item, lookup)?; - } - } - Value::Null | Value::Bool(_) | Value::Number(_) => {} - } - Ok(()) -} - -fn interpolate_string(input: &str, lookup: &impl Fn(&str) -> Option) -> Result { - let mut output = String::new(); - let mut rest = input; - - while let Some(start) = rest.find("${") { - output.push_str(&rest[..start]); - let after_start = &rest[start + 2..]; - let Some(end) = after_start.find('}') else { - return Err(SwitchyardError::InvalidConfig(format!( - "unterminated environment variable in `{input}`" - ))); - }; - let name = &after_start[..end]; - if name.is_empty() { - return Err(SwitchyardError::InvalidConfig( - "empty environment variable reference".to_string(), - )); - } - let value = lookup(name).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "environment variable {name} is not set for profile config interpolation" - )) - })?; - output.push_str(&value); - rest = &after_start[end + 1..]; - } - - output.push_str(rest); - Ok(output) -} diff --git a/crates/switchyard-components-v2/src/config/resolving.rs b/crates/switchyard-components-v2/src/config/resolving.rs deleted file mode 100644 index b019e0c7..00000000 --- a/crates/switchyard-components-v2/src/config/resolving.rs +++ /dev/null @@ -1,351 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Resolution of parsed profile configs into runtime-ready objects. - -use std::{collections::BTreeMap, fmt}; - -use serde_json::Value; -use switchyard_core::{ - EndpointConfig, EndpointId, LlmTarget, LlmTargetId, ProfileId, Result, SwitchyardError, -}; - -use super::parsing::{ProfileConfigDocument, SerializedProfileConfig, TargetConfig}; -use crate::profiles::{ProfileConfigEntry, SubagentOverrideProfile}; -use crate::{Profile, ProfileHooks}; - -impl ProfileConfigDocument { - /// Resolves endpoints and validates every profile through its owning profile type. - pub fn resolve(&self) -> Result { - let targets = self.resolve_targets()?; - let (profiles, profile_subagent_targets) = self.resolve_profiles(&targets)?; - Ok(ProfileConfigPlan { - targets, - profiles, - profile_subagent_targets, - }) - } - - // Resolves all file-facing targets before profile-specific config expansion. - fn resolve_targets(&self) -> Result> { - let mut targets = BTreeMap::new(); - for (target_id, target) in &self.targets { - targets.insert( - target_id.clone(), - target.resolve(target_id, &self.endpoints)?, - ); - } - Ok(targets) - } - - // Parses profile-specific config bodies into the generated typed profile enum. - // Also collects resolved sub-agent targets for profiles that declare one. - fn resolve_profiles( - &self, - targets: &BTreeMap, - ) -> Result<( - BTreeMap, - BTreeMap, - )> { - let env = ProfileBuildEnv::new(targets); - let mut profiles = BTreeMap::new(); - let mut subagent_targets: BTreeMap = BTreeMap::new(); - for (profile_id, profile) in &self.profiles { - // The envelope-level sub-agent override must name a resolvable target; - // an unknown reference is a startup configuration error. - if let Some(target_id) = profile.subagent_target() { - let target = targets.get(target_id).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "profile {profile_id}: subagent_target references unknown target {}", - target_id.as_str() - )) - })?; - subagent_targets.insert(profile_id.clone(), target.clone()); - } - profiles.insert( - profile_id.clone(), - resolve_profile(profile_id, profile, &env)?, - ); - } - Ok((profiles, subagent_targets)) - } -} - -fn resolve_profile( - profile_id: &ProfileId, - profile: &SerializedProfileConfig, - env: &ProfileBuildEnv<'_>, -) -> Result { - crate::profiles::parse_profile_config(profile.profile_type(), profile.body().clone(), env) - .map_err(|error| SwitchyardError::InvalidConfig(format!("profile {profile_id}: {error}"))) -} - -/// Target lookup environment used while profile-owned configs are parsed. -pub struct ProfileBuildEnv<'a> { - targets: &'a BTreeMap, -} - -impl<'a> ProfileBuildEnv<'a> { - /// Creates a build environment over already-resolved targets. - pub fn new(targets: &'a BTreeMap) -> Self { - Self { targets } - } - - /// Resolves a profile-local target ID into a concrete runtime target. - pub fn target(&self, target_id: &LlmTargetId) -> Result<&'a LlmTarget> { - self.targets.get(target_id).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!("profile references unknown target {target_id}")) - }) - } -} - -/// Trait implemented by the profile config macro for profile-owned parsing. -pub trait ProfileConfigDefinition: Sized { - /// Stable discriminator used in the file-facing `type` field. - const PROFILE_TYPE: &'static str; - - /// Parses profile-owned fields and resolves any `#[profile_target]` references. - fn parse_profile_config(value: Value, env: &ProfileBuildEnv<'_>) -> Result; -} - -/// Public contract implemented by every profile config type. -/// -/// The `profile_config` attribute macro generates parsing metadata, but the -/// runtime build remains explicit here so profile authors can construct -/// backends, pollers, stats handles, and validation in normal Rust. -pub trait ProfileConfig: ProfileConfigDefinition { - /// Runtime profile produced by this config. - type Runtime: Profile + ProfileHooks + 'static; - - /// Builds this config into its runtime profile. - fn build(&self) -> Result; - - /// Builds this config into the object-safe profile used by config plans. - fn build_boxed(&self) -> Result> { - Ok(Box::new(self.build()?)) - } -} - -impl TargetConfig { - // Resolves endpoint inheritance while preserving the map key as the target ID. - fn resolve( - &self, - target_id: &LlmTargetId, - endpoints: &BTreeMap, - ) -> Result { - let inherited = match &self.endpoint { - Some(endpoint_id) => endpoints.get(endpoint_id).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "target {target_id} references unknown endpoint {endpoint_id}" - )) - })?, - None => &EndpointConfig::default(), - }; - let overrides = EndpointConfig { - base_url: self.base_url.clone(), - api_key: self.api_key.clone(), - timeout_secs: self.timeout_secs, - }; - Ok(LlmTarget { - id: target_id.clone(), - model: self.model.clone(), - format: self.format, - endpoint: inherited.with_overrides(&overrides), - extra_body: self.extra_body.clone(), - extra_headers: self.extra_headers.clone(), - }) - } -} - -/// Validated profile config plan ready to build runtime profiles. -/// -/// This type contains resolved targets and typed profile config bodies. It -/// deliberately does not contain built `Profile` runtimes; call -/// `build_profile` or `build_profiles` when runtime ownership is needed. -#[derive(Clone, PartialEq)] -pub struct ProfileConfigPlan { - /// Resolved targets keyed by the IDs used in the parsed config. - targets: BTreeMap, - /// Typed profile configs keyed by the user-facing profile ID. - profiles: BTreeMap, - /// Resolved sub-agent override targets for profiles that declare one. - profile_subagent_targets: BTreeMap, -} - -impl fmt::Debug for ProfileConfigPlan { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - // Avoid printing resolved target/profile configs because they can carry API keys. - let profile_types = self - .profiles - .iter() - .map(|(profile_id, profile)| (profile_id, profile.profile_type())) - .collect::>(); - formatter - .debug_struct("ProfileConfigPlan") - .field("targets", &self.targets.keys().collect::>()) - .field("profiles", &profile_types) - .finish() - } -} - -impl ProfileConfigPlan { - /// Returns the number of resolved targets in the plan. - pub fn target_count(&self) -> usize { - self.targets.len() - } - - /// Returns the number of validated profile configs in the plan. - pub fn profile_count(&self) -> usize { - self.profiles.len() - } - - /// Returns a resolved target by ID. - pub fn target(&self, target_id: &LlmTargetId) -> Option<&LlmTarget> { - self.targets.get(target_id) - } - - /// Iterates resolved targets in deterministic target-ID order. - pub fn targets(&self) -> impl Iterator { - self.targets.iter() - } - - /// Iterates profile IDs in deterministic profile-ID order. - pub fn profile_ids(&self) -> impl Iterator { - self.profiles.keys() - } - - /// Returns the serialized type discriminator for one profile config. - pub fn profile_type(&self, profile_id: &ProfileId) -> Option<&str> { - self.profiles - .get(profile_id) - .map(ProfileConfigEntry::profile_type) - } - - /// Builds one profile runtime by profile ID. - pub fn build_profile(&self, profile_id: &ProfileId) -> Result> { - let profile = self.profiles.get(profile_id).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!("unknown profile {profile_id}")) - })?; - self.build_typed_profile(profile_id, profile) - } - - /// Builds every profile in the document into an object-safe runtime profile. - pub fn build_profiles(&self) -> Result>> { - let mut profiles = BTreeMap::new(); - for (profile_id, profile) in &self.profiles { - profiles.insert( - profile_id.clone(), - self.build_typed_profile(profile_id, profile)?, - ); - } - Ok(profiles) - } - - /// Returns every resolved target as a directly addressable serving target. - pub fn exposed_targets(&self) -> impl Iterator { - self.targets.values() - } - - // Builds one typed profile config with profile-ID context in errors. - // Wraps with SubagentOverrideProfile when the profile declares a subagent_target. - fn build_typed_profile( - &self, - profile_id: &ProfileId, - profile: &ProfileConfigEntry, - ) -> Result> { - let inner = profile.build_boxed().map_err(|error| { - SwitchyardError::InvalidConfig(format!("profile {profile_id}: {error}")) - })?; - if let Some(target) = self.profile_subagent_targets.get(profile_id) { - SubagentOverrideProfile::new(inner, target.clone()) - .map(|p| Box::new(p) as Box) - .map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "profile {profile_id}: subagent override: {error}" - )) - }) - } else { - Ok(inner) - } - } -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use async_trait::async_trait; - use serde_json::json; - use switchyard_core::{BackendFormat, ChatResponse, ModelId}; - - use super::*; - - #[switchyard_components_v2_macros::profile_config("optional-target-test")] - struct OptionalTargetConfig { - /// Optional target used to test macro-generated `Option` resolution. - #[profile_target] - pub target: Option, - } - - struct OptionalTargetProfile; - - impl ProfileConfig for OptionalTargetConfig { - type Runtime = OptionalTargetProfile; - - fn build(&self) -> Result { - Ok(OptionalTargetProfile) - } - } - - #[async_trait] - impl crate::ProfileHooks for OptionalTargetProfile { - type ProcessedRequest = crate::ProfileInput; - - async fn process(&self, input: crate::ProfileInput) -> Result { - Ok(input) - } - - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } - } - - #[async_trait] - impl crate::Profile for OptionalTargetProfile { - async fn run(&self, input: crate::ProfileInput) -> Result { - Ok(ChatResponse::openai_completion(json!({ - "model": input.request.model(), - "choices": [] - })) - .into()) - } - } - - fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) - } - - #[test] - fn optional_target_fields_are_resolved_by_the_profile_config_macro() -> Result<()> { - let weak = target("weak", "weak/model")?; - let targets = BTreeMap::from([(weak.id.clone(), weak)]); - let env = ProfileBuildEnv::new(&targets); - - let with_target = - OptionalTargetConfig::parse_profile_config(json!({"target": "weak"}), &env)?; - assert_eq!( - with_target.target.map(|target| target.id), - Some(LlmTargetId::new("weak")?) - ); - - let without_target = OptionalTargetConfig::parse_profile_config(json!({}), &env)?; - assert_eq!(without_target.target, None); - Ok(()) - } -} diff --git a/crates/switchyard-components-v2/src/lib.rs b/crates/switchyard-components-v2/src/lib.rs deleted file mode 100644 index bfeaa30a..00000000 --- a/crates/switchyard-components-v2/src/lib.rs +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Experimental components-v2 crate for the flatter profile-owned design. -//! -//! This crate is intentionally separate from `switchyard-components` so the -//! rewrite shape can evolve without contaminating the existing production config surface. - -extern crate self as switchyard_components_v2; - -mod backend; -mod config; -mod profile; -pub mod profiles; -mod stats; -mod stats_recording; - -pub use config::{ - parse_profile_config_path, parse_profile_config_str, parse_profile_config_str_with_env_lookup, - ProfileConfig, ProfileConfigDocument, ProfileConfigFormat, ProfileConfigPlan, -}; -pub use profile::{ - Profile, ProfileHooks, ProfileInput, ProfileResponse, RequestMetadata, RoutingMetadata, -}; -pub use profiles::{ - EndpointHealth, EndpointHealthStatus, LatencyServiceProcessedRequest, LatencyServiceProfile, - LatencyServiceProfileConfig, LlmRoutingDecision, LlmRoutingProcessedRequest, LlmRoutingProfile, - LlmRoutingProfileConfig, LlmRoutingTierMapping, NoopProfile, NoopProfileConfig, - PassthroughProfile, PassthroughProfileConfig, RandomRoutingProcessedRequest, - RandomRoutingProfile, RandomRoutingProfileConfig, SelectedTarget, StageRouterClassifierConfig, - StageRouterDecision, StageRouterDecisionSource, StageRouterPickerMode, - StageRouterProcessedRequest, StageRouterProfile, StageRouterProfileConfig, StageRouterTier, -}; -pub use stats::profile_stats_accumulator; -pub use switchyard_components_v2_macros::profile_config; - -/// Implementation details used by generated profile-config code. -/// -/// This namespace is public only so proc-macro expansion has a stable path. -/// It is not part of the user-facing components-v2 API. -#[doc(hidden)] -pub mod __private { - pub use crate::config::{ProfileBuildEnv, ProfileConfigDefinition}; -} diff --git a/crates/switchyard-components-v2/src/profile.rs b/crates/switchyard-components-v2/src/profile.rs deleted file mode 100644 index 396ef580..00000000 --- a/crates/switchyard-components-v2/src/profile.rs +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Profile runtime contracts for components-v2. - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -use switchyard_core::{ChatRequest, ChatRequestType, ChatResponse, RequestId, Result}; - -/// Explicit per-request metadata passed to v2 profiles. -/// -/// Header keys are stored as lowercase ASCII names. Repeated header values are -/// preserved in the original value order because endpoints may need to pass -/// through multi-value headers without silently collapsing them. -#[derive(Clone, Default, Eq, PartialEq)] -pub struct RequestMetadata { - /// Caller-provided request identifier, when present. - pub request_id: Option, - /// Inbound wire format recorded by the endpoint, when present. - pub inbound_format: Option, - /// Request headers supplied by the caller, keyed by normalized header name. - pub headers: BTreeMap>, -} - -/// Input object handed to v2 profiles. -/// -/// This is the only request carrier in the v2 profile path. It owns the -/// provider-neutral [`ChatRequest`] plus endpoint-supplied metadata such as -/// request IDs, inbound format, and headers. Policy decisions remain local to -/// the profile implementation instead of being hidden in this input object. -#[derive(Clone, PartialEq)] -pub struct ProfileInput { - /// Provider-neutral chat request entering the profile. - pub request: ChatRequest, - /// Endpoint-supplied request metadata. - pub metadata: RequestMetadata, -} - -/// Routing decision metadata emitted by profiles that select among targets. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct RoutingMetadata { - /// Upstream model selected by the router. - pub selected_model: Option, - /// Routing tier selected by the router. - pub selected_tier: Option, - /// Router confidence for the selected decision. - pub confidence: Option, - /// Stable router implementation/version label. - pub router_version: Option, - /// Router tolerance, probability, or decision threshold. - pub tolerance: Option, - /// Short human-readable route rationale. - pub rationale: Option, -} - -impl RoutingMetadata { - /// Returns true when no metadata field is set. - pub fn is_empty(&self) -> bool { - self.selected_model.is_none() - && self.selected_tier.is_none() - && self.confidence.is_none() - && self.router_version.is_none() - && self.tolerance.is_none() - && self.rationale.is_none() - } -} - -/// Full profile result returned to servers and other erased-profile callers. -pub struct ProfileResponse { - /// Final backend response after profile response processing. - pub response: ChatResponse, - /// Optional routing metadata for response headers and audits. - pub routing_metadata: Option, -} - -impl ProfileResponse { - /// Creates a profile response without routing metadata. - pub fn new(response: ChatResponse) -> Self { - Self { - response, - routing_metadata: None, - } - } - - /// Creates a profile response with routing metadata. - pub fn with_routing_metadata( - response: ChatResponse, - routing_metadata: RoutingMetadata, - ) -> Self { - Self { - response, - routing_metadata: (!routing_metadata.is_empty()).then_some(routing_metadata), - } - } - - /// Splits this value into the final response and optional routing metadata. - pub fn into_parts(self) -> (ChatResponse, Option) { - (self.response, self.routing_metadata) - } - - /// Returns the response body when this is a buffered response. - pub fn body(&self) -> Option<&serde_json::Value> { - self.response.body() - } -} - -impl From for ProfileResponse { - fn from(response: ChatResponse) -> Self { - Self::new(response) - } -} - -/// Object-safe runtime surface for a complete profile. -/// -/// Servers and config-built profile maps use this trait when they need to run -/// an entire profile and receive a final response. It intentionally exposes -/// only `run()` so dynamic dispatch does not erase profile-specific processed -/// state returned by hook-level APIs. -#[async_trait] -pub trait Profile: Send + Sync { - /// Executes the complete profile flow and returns the final response. - async fn run(&self, input: ProfileInput) -> Result; -} - -/// Typed hook surface for profile authors and embedders. -/// -/// These methods support middleware-style integrations that want Switchyard to -/// prepare a request and/or process a response without owning the transport. -/// The associated `ProcessedRequest` type is a profile-owned struct, which lets -/// profiles expose real per-call state, such as a routing decision, without a -/// generic side channel. -#[async_trait] -pub trait ProfileHooks: Send + Sync { - /// Profile-owned request-side state. - type ProcessedRequest: Send + Sync; - - /// Runs the profile's request-side hook. - async fn process(&self, input: ProfileInput) -> Result; - - /// Runs the profile's response-side hook after a backend response exists. - async fn rprocess( - &self, - processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result; -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use serde_json::json; - use switchyard_core::ChatRequestType; - - use super::*; - - #[test] - fn request_metadata_is_plain_data() { - let metadata = RequestMetadata { - request_id: None, - inbound_format: Some(ChatRequestType::OpenAiChat), - headers: BTreeMap::from([( - "x-switchyard-trace".to_string(), - vec!["abc123".to_string()], - )]), - }; - - assert_eq!( - metadata - .headers - .get("x-switchyard-trace") - .map(Vec::as_slice), - Some(&["abc123".to_string()][..]) - ); - assert_eq!(metadata.inbound_format, Some(ChatRequestType::OpenAiChat)); - } - - #[test] - fn profile_input_is_plain_data() { - let input = ProfileInput { - request: ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })), - metadata: RequestMetadata { - request_id: None, - inbound_format: None, - headers: BTreeMap::from([( - "x-request-source".to_string(), - vec!["unit-test".to_string()], - )]), - }, - }; - - assert_eq!(input.request.model(), Some("client/model")); - assert_eq!( - input - .metadata - .headers - .get("x-request-source") - .map(Vec::as_slice), - Some(&["unit-test".to_string()][..]) - ); - assert!(input.metadata.request_id.is_none()); - } -} diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/health.rs b/crates/switchyard-components-v2/src/profiles/latency_service/health.rs deleted file mode 100644 index ecfc6cff..00000000 --- a/crates/switchyard-components-v2/src/profiles/latency_service/health.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Health snapshots used by the components-v2 latency-service profile. - -use serde::{Deserialize, Serialize}; - -/// Endpoint states returned by the external Latency Service. -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EndpointHealthStatus { - /// Endpoint is eligible for preferred routing. - Healthy, - /// Endpoint can still receive traffic, but only after healthy and unknown pools. - Degraded, - /// Endpoint has no current health verdict. - Unknown, -} - -impl EndpointHealthStatus { - /// Routing preference order for latency-service target selection. - pub(crate) const ROUTING_ORDER: [Self; 3] = [Self::Healthy, Self::Unknown, Self::Degraded]; - - /// Stable lowercase label used by routing metadata. - pub fn as_str(self) -> &'static str { - match self { - Self::Healthy => "healthy", - Self::Degraded => "degraded", - Self::Unknown => "unknown", - } - } -} - -/// Cached health snapshot for one configured target. -#[derive(Clone, Copy, Debug, PartialEq, Serialize)] -pub struct EndpointHealth { - /// Service-reported health status for the endpoint. - pub status: EndpointHealthStatus, - /// Optional last observed latency in milliseconds. - #[serde(skip_serializing_if = "Option::is_none")] - pub last_latency_ms: Option, -} - -impl EndpointHealth { - /// Creates a health snapshot with no latency sample. - pub fn new(status: EndpointHealthStatus) -> Self { - Self { - status, - last_latency_ms: None, - } - } - - /// Creates a health snapshot with a latency sample. - pub fn with_latency(status: EndpointHealthStatus, last_latency_ms: f64) -> Self { - Self { - status, - last_latency_ms: Some(last_latency_ms), - } - } -} diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs b/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs deleted file mode 100644 index 2242248d..00000000 --- a/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs +++ /dev/null @@ -1,390 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Latency-service profile implemented as a profile-owned runtime. - -mod health; -mod polling; -mod selection; - -#[cfg(test)] -mod tests; - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Instant; - -use async_trait::async_trait; -use parking_lot::RwLock; -use switchyard_components::stats::usage_from_body; -use switchyard_components::StatsAccumulator; -use switchyard_core::{ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError}; - -pub use self::health::{EndpointHealth, EndpointHealthStatus}; -use self::polling::{duration_from_secs, HealthPoller, HealthResponse}; -use self::selection::select_target; -pub use self::selection::SelectedTarget; -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::{ - profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse, - RoutingMetadata, -}; - -const DEFAULT_POLL_TIMEOUT_SECS: f64 = 5.0; -const DEFAULT_MAX_RETRIES: usize = 2; - -/// Default HTTP timeout for latency-service health polls. -fn default_poll_timeout_secs() -> f64 { - DEFAULT_POLL_TIMEOUT_SECS -} - -/// Default number of alternate targets to try after the first failed call. -fn default_max_retries() -> usize { - DEFAULT_MAX_RETRIES -} - -/// Config for the flatter latency-service profile. -/// -/// The profile owns the health cache and exposes [`LatencyServiceProfile::poll_once`] -/// for whichever control plane embeds it. It does not spawn a background task; -/// `run()` and `process()` perform a guarded initial refresh for multi-target -/// profiles and then route from the profile-owned cache. -#[profile_config("latency-service")] -pub struct LatencyServiceProfileConfig { - /// Base URL for the external latency service. - pub latency_service_url: String, - /// Concrete targets this profile may select at request time. - #[profile_target] - pub targets: Vec, - /// Per-poll HTTP timeout in seconds. - #[serde(default = "default_poll_timeout_secs")] - pub poll_timeout_secs: f64, - /// Number of alternate target attempts after the first failed call. - #[serde(default = "default_max_retries")] - pub max_retries: usize, -} - -impl ProfileConfig for LatencyServiceProfileConfig { - type Runtime = LatencyServiceProfile; - - /// Builds the runtime profile using existing native backend construction. - fn build(&self) -> Result { - validate_config(self)?; - let target_ids = self - .targets - .iter() - .map(|target| target.id.clone()) - .collect::>(); - let poll_timeout = duration_from_secs(self.poll_timeout_secs, "poll_timeout_secs")?; - let poller = - HealthPoller::new(&self.latency_service_url, target_ids.clone(), poll_timeout)?; - - let mut backends = BTreeMap::new(); - let mut health = BTreeMap::new(); - for target in &self.targets { - backends.insert(target.id.clone(), native_target_backend(target.clone())?); - health.insert( - target.id.clone(), - EndpointHealth::new(EndpointHealthStatus::Unknown), - ); - } - - Ok(LatencyServiceProfile { - poller, - backends, - health: RwLock::new(health), - poll_count: AtomicU64::new(0), - initial_refresh_in_flight: AtomicBool::new(false), - max_retries: self.max_retries, - stats: profile_stats_accumulator(), - }) - } -} - -/// Latency-service profile in the flatter design. -pub struct LatencyServiceProfile { - poller: HealthPoller, - backends: BTreeMap, - health: RwLock>, - poll_count: AtomicU64, - initial_refresh_in_flight: AtomicBool, - max_retries: usize, - stats: StatsAccumulator, -} - -/// Processed latency-service request with the target selected for response processing. -pub struct LatencyServiceProcessedRequest { - /// Routed input prepared for the selected backend. - pub profile_input: ProfileInput, - /// Target associated with the backend response passed to `rprocess()`. - pub selected: SelectedTarget, -} - -impl LatencyServiceProfile { - /// Returns true once at least one explicit health poll has completed successfully. - pub fn is_ready(&self) -> bool { - self.poll_count.load(Ordering::Relaxed) > 0 - } - - /// Returns a point-in-time copy of the cached health table. - pub fn health_snapshot(&self) -> BTreeMap { - self.health.read().clone() - } - - /// Applies externally supplied health for a configured target. - /// - /// This is useful for tests and control-plane integrations that already own health - /// polling and want the profile hot path to remain network-free. - pub fn update_health(&self, target_id: LlmTargetId, health: EndpointHealth) -> Result<()> { - let mut cache = self.health.write(); - let Some(entry) = cache.get_mut(&target_id) else { - return Err(SwitchyardError::InvalidConfig(format!( - "latency_service target {target_id} is not configured" - ))); - }; - *entry = health; - Ok(()) - } - - /// Performs one latency-service health poll and updates the local cache. - /// - /// A failed poll resets known health to `unknown` so stale healthy readings do not - /// keep biasing target selection after the control plane becomes unavailable. - pub async fn poll_once(&self) -> Result<()> { - match self - .poller - .fetch_health() - .await - .and_then(|payload| self.apply_health(payload)) - { - Ok(()) => { - self.poll_count.fetch_add(1, Ordering::Relaxed); - Ok(()) - } - Err(error) => { - self.reset_to_unknown(); - Err(error) - } - } - } - - async fn refresh_if_unready(&self) { - if self.backends.len() <= 1 || self.is_ready() || self.has_health_signal() { - return; - } - if self - .initial_refresh_in_flight - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - return; - } - - if let Err(error) = self.poll_once().await { - tracing::warn!( - error = %error, - "latency_service initial health refresh failed; routing with unknown health" - ); - } - self.initial_refresh_in_flight - .store(false, Ordering::Release); - } - - fn has_health_signal(&self) -> bool { - self.health.read().values().any(|health| { - health.status != EndpointHealthStatus::Unknown || health.last_latency_ms.is_some() - }) - } - - // Selects the target and rewrites the request model without persisting state. - fn route_request(&self, mut input: ProfileInput) -> Result { - let selected = self.select(&BTreeSet::new())?; - let backend = self.selected_backend(&selected.target_id)?; - input.request.set_model(backend.target().model.as_str()); - Ok(LatencyServiceProcessedRequest { - profile_input: input, - selected, - }) - } - - // Reads the health cache and applies the latency-service tiering policy. - fn select(&self, excluded: &BTreeSet) -> Result { - let snapshot = self.health.read(); - select_target(&snapshot, excluded) - } - - // Finds the concrete backend for a selected target. - fn selected_backend(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> { - self.backends.get(target_id).ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "latency_service selected target {target_id} without a configured backend" - )) - }) - } - - // Resets preference signals while preserving the configured target set. - fn reset_to_unknown(&self) { - let mut cache = self.health.write(); - for health in cache.values_mut() { - *health = EndpointHealth::new(EndpointHealthStatus::Unknown); - } - } - - // Applies health for known targets and resets omitted known targets to unknown. - fn apply_health(&self, payload: HealthResponse) -> Result<()> { - let known_ids = self - .poller - .target_ids() - .iter() - .map(LlmTargetId::as_str) - .collect::>(); - let mut reported = BTreeSet::new(); - let mut cache = self.health.write(); - for (target_id, health) in payload.endpoint_health { - if known_ids.contains(target_id.as_str()) { - let target_id = LlmTargetId::new(target_id)?; - if let Some(entry) = cache.get_mut(&target_id) { - *entry = health; - reported.insert(target_id); - } - } - } - for (target_id, health) in cache.iter_mut() { - if !reported.contains(target_id) { - *health = EndpointHealth::new(EndpointHealthStatus::Unknown); - } - } - Ok(()) - } - - fn routing_metadata(&self, selected: &SelectedTarget, selected_model: &str) -> RoutingMetadata { - RoutingMetadata { - selected_model: Some(selected_model.to_string()), - selected_tier: Some(selected.health_status.as_str().to_string()), - confidence: None, - router_version: Some("latency-service:v1".to_string()), - tolerance: None, - rationale: Some(format!( - "latency service selected target {} from {} health tier", - selected.target_id, - selected.health_status.as_str() - )), - } - } -} - -#[async_trait] -impl ProfileHooks for LatencyServiceProfile { - type ProcessedRequest = LatencyServiceProcessedRequest; - - /// Performs a standalone latency-service routing rewrite for hook-level inspection. - async fn process(&self, input: ProfileInput) -> Result { - self.refresh_if_unready().await; - self.route_request(input) - } - - /// Leaves the backend response unchanged after latency-aware routing completes. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for LatencyServiceProfile { - /// Executes latency-aware routing with retry state local to this call. - async fn run(&self, mut input: ProfileInput) -> Result { - self.refresh_if_unready().await; - let profile_started_at = Instant::now(); - let mut tried = BTreeSet::new(); - let mut last_error = None; - - for _ in 0..=self.max_retries { - if tried.len() == self.backends.len() { - break; - } - let selected = match self.select(&tried) { - Ok(selected) => selected, - Err(_) if last_error.is_some() => break, - Err(error) => return Err(error), - }; - tried.insert(selected.target_id.clone()); - let selected_backend = self.selected_backend(&selected.target_id)?; - let stats_model = selected_backend.target().model.to_string(); - let routing_metadata = self.routing_metadata(&selected, &stats_model); - input - .request - .set_model(selected_backend.target().model.as_str()); - let backend_started_at = Instant::now(); - - match selected_backend.call(&input.request).await { - Ok(response) => { - let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; - self.stats.record_success( - stats_model.clone(), - Some(backend_latency_ms), - None, - )?; - let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0; - let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0); - let usage = response.body().map(usage_from_body).unwrap_or_default(); - self.stats.record_usage_after_success_attribution( - stats_model, - usage, - Some(total_latency_ms), - Some(routing_overhead_ms), - None, - )?; - let processed = LatencyServiceProcessedRequest { - profile_input: input, - selected, - }; - let response = self.rprocess(&processed, response).await?; - return Ok(ProfileResponse::with_routing_metadata( - response, - routing_metadata, - )); - } - Err(error) => { - self.stats.record_error(stats_model, None)?; - last_error = Some(error); - } - } - } - - Err(last_error.unwrap_or_else(|| { - SwitchyardError::Backend( - "latency_service profile exhausted retries without calling a target".to_string(), - ) - })) - } -} - -fn validate_config(config: &LatencyServiceProfileConfig) -> Result<()> { - if config.latency_service_url.trim().is_empty() { - return Err(SwitchyardError::InvalidConfig( - "latency_service profile requires latency_service_url".to_string(), - )); - } - if config.targets.is_empty() { - return Err(SwitchyardError::InvalidConfig( - "latency_service profile requires at least one target".to_string(), - )); - } - duration_from_secs(config.poll_timeout_secs, "poll_timeout_secs")?; - - let mut seen = BTreeSet::new(); - for target in &config.targets { - if !seen.insert(target.id.clone()) { - return Err(SwitchyardError::InvalidConfig(format!( - "latency_service profile has duplicate target {}", - target.id - ))); - } - } - Ok(()) -} diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs b/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs deleted file mode 100644 index 280d09f6..00000000 --- a/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Health polling helpers for the components-v2 latency-service profile. - -use std::collections::BTreeMap; -use std::time::Duration; - -use serde::Deserialize; -use switchyard_core::{LlmTargetId, Result, SwitchyardError}; - -use super::{EndpointHealth, EndpointHealthStatus}; - -/// Polling client for one latency-service profile. -pub(crate) struct HealthPoller { - url: String, - target_ids: Vec, - client: reqwest::Client, -} - -impl HealthPoller { - /// Creates a poller for the profile's configured target IDs. - pub(crate) fn new( - latency_service_url: &str, - target_ids: Vec, - poll_timeout: Duration, - ) -> Result { - let url = format!( - "{}/v1/endpoints/health", - latency_service_url.trim_end_matches('/') - ); - let client = reqwest::Client::builder() - .timeout(poll_timeout) - .user_agent("SwitchyardLatencyServiceProfile") - .build() - .map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "latency_service failed to build HTTP client: {error}" - )) - })?; - Ok(Self { - url, - target_ids, - client, - }) - } - - /// Returns the target IDs this poller sends to the latency service. - pub(crate) fn target_ids(&self) -> &[LlmTargetId] { - &self.target_ids - } - - /// Fetches one health payload from the external latency service. - pub(crate) async fn fetch_health(&self) -> Result { - let params = self - .target_ids - .iter() - .map(|target_id| ("endpoint_ids", target_id.as_str())) - .collect::>(); - let response = self - .client - .get(&self.url) - .query(¶ms) - .send() - .await - .map_err(|error| { - SwitchyardError::Upstream(format!("Latency Service health request failed: {error}")) - })?; - let status = response.status(); - if !status.is_success() { - return Err(SwitchyardError::Upstream(format!( - "Latency Service health request returned HTTP {status}" - ))); - } - response - .json::() - .await - .map(HealthResponse::from) - .map_err(|error| { - SwitchyardError::Upstream(format!( - "Latency Service health response was invalid JSON: {error}" - )) - }) - } -} - -/// Parsed health response with public target health values. -pub(crate) struct HealthResponse { - pub(crate) endpoint_health: BTreeMap, -} - -#[derive(Debug, Deserialize)] -struct RawHealthResponse { - endpoint_health: BTreeMap, -} - -#[derive(Debug, Deserialize)] -struct HealthEntry { - status: EndpointHealthStatus, - #[serde(default)] - last_latency_ms: Option, -} - -impl From for HealthResponse { - fn from(response: RawHealthResponse) -> Self { - Self { - endpoint_health: response - .endpoint_health - .into_iter() - .map(|(target_id, health)| (target_id, health.into())) - .collect(), - } - } -} - -impl From for EndpointHealth { - fn from(entry: HealthEntry) -> Self { - Self { - status: entry.status, - last_latency_ms: entry.last_latency_ms, - } - } -} - -/// Converts positive finite seconds into a Rust duration. -pub(crate) fn duration_from_secs(value: f64, field: &'static str) -> Result { - if !value.is_finite() || value <= 0.0 { - return Err(SwitchyardError::InvalidConfig(format!( - "latency_service {field} must be finite and positive, got {value:?}" - ))); - } - Duration::try_from_secs_f64(value).map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "latency_service {field} is outside the supported duration range: {error}" - )) - }) -} diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs b/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs deleted file mode 100644 index 02c6bd3c..00000000 --- a/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Target selection policy for the components-v2 latency-service profile. - -use std::collections::{BTreeMap, BTreeSet}; - -use rand::distributions::{Distribution, WeightedIndex}; -use rand::Rng; -use serde::{Deserialize, Serialize}; -use switchyard_core::{LlmTargetId, Result, SwitchyardError}; - -use super::{EndpointHealth, EndpointHealthStatus}; - -/// Selected target ID emitted by the latency-service selector. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct SelectedTarget { - /// Target ID selected from the health cache. - pub target_id: LlmTargetId, - /// Health tier that made the target selectable. - pub health_status: EndpointHealthStatus, -} - -/// Selects a target from the health cache while avoiding excluded targets when possible. -pub(crate) fn select_target( - snapshot: &BTreeMap, - excluded: &BTreeSet, -) -> Result { - for tier in EndpointHealthStatus::ROUTING_ORDER { - let entries = candidates_for_tier(snapshot, excluded, tier); - if !entries.is_empty() { - return pick_from_tier(&entries, tier); - } - } - - Err(SwitchyardError::InvalidConfig( - "latency_service profile has no selectable targets".to_string(), - )) -} - -fn candidates_for_tier<'a>( - snapshot: &'a BTreeMap, - excluded: &BTreeSet, - tier: EndpointHealthStatus, -) -> Vec<(&'a LlmTargetId, EndpointHealth)> { - snapshot - .iter() - .filter_map(|(target_id, health)| { - (!excluded.contains(target_id) && health.status == tier).then_some((target_id, *health)) - }) - .collect() -} - -fn pick_from_tier( - entries: &[(&LlmTargetId, EndpointHealth)], - health_status: EndpointHealthStatus, -) -> Result { - if entries.len() == 1 { - return Ok(SelectedTarget { - target_id: entries[0].0.clone(), - health_status, - }); - } - - let weights = entries - .iter() - .map(|(_, health)| { - health - .last_latency_ms - .filter(|value| value.is_finite() && *value > 0.0) - .map(|value| 1.0 / value) - }) - .collect::>>(); - - let index = if let Some(weights) = weights { - weighted_index(&weights)? - } else { - rand::thread_rng().gen_range(0..entries.len()) - }; - - Ok(SelectedTarget { - target_id: entries[index].0.clone(), - health_status, - }) -} - -fn weighted_index(weights: &[f64]) -> Result { - let distribution = WeightedIndex::new(weights).map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "latency_service computed invalid target weights: {error}" - )) - })?; - Ok(distribution.sample(&mut rand::thread_rng())) -} diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs b/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs deleted file mode 100644 index 0bdd8ed0..00000000 --- a/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs +++ /dev/null @@ -1,893 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Unit tests for the components-v2 latency-service profile internals. - -use std::collections::{BTreeMap, BTreeSet}; -use std::io::{BufRead, BufReader, Write}; -use std::net::TcpListener; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicU64}; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; - -use async_trait::async_trait; -use futures_core::Stream; -use parking_lot::RwLock; -use serde_json::{json, Value}; -use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent}; - -use super::polling::HealthPoller; -use super::selection::select_target; -use super::*; -use crate::backend::ProfileBackend; -use crate::{ProfileInput, RequestMetadata}; - -#[derive(Clone, Debug, PartialEq)] -struct ObservedCall { - backend: String, - body: Value, -} - -struct TestBackend { - name: String, - calls: Arc>>, - error: Option<&'static str>, - response_mode: ResponseMode, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ResponseMode { - Completion, - OpenAiStream, -} - -struct OneEventStream { - event: Option>, -} - -impl Stream for OneEventStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.event.take()) - } -} - -#[async_trait] -impl ProfileBackend for TestBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.calls - .lock() - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))? - .push(ObservedCall { - backend: self.name.clone(), - body: request.body().clone(), - }); - if let Some(error) = self.error { - return Err(SwitchyardError::Backend(error.to_string())); - } - match self.response_mode { - ResponseMode::Completion => Ok(ChatResponse::openai_completion(json!({ - "backend": self.name, - "model": request.model(), - "usage": { - "prompt_tokens": 13, - "completion_tokens": 5, - }, - }))), - ResponseMode::OpenAiStream => { - Ok(ChatResponse::OpenAiStream(Box::pin(OneEventStream { - event: Some(Ok(StreamEvent::Json(json!({ - "backend": self.name, - "model": request.model(), - })))), - }))) - } - } - } -} - -#[derive(Clone, Debug, PartialEq)] -struct CapturedRequest { - path: String, -} - -struct OneShotServer { - base_url: String, - captured: Arc>>, - handle: Option>>, -} - -impl OneShotServer { - fn json(status: u16, body: Value) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?; - let address = listener - .local_addr() - .map_err(|error| SwitchyardError::Other(format!("local addr failed: {error}")))?; - let captured = Arc::new(Mutex::new(None)); - let captured_for_thread = Arc::clone(&captured); - let handle = thread::spawn(move || serve_once(listener, status, body, captured_for_thread)); - Ok(Self { - base_url: format!("http://{address}"), - captured, - handle: Some(handle), - }) - } - - fn base_url(&self) -> String { - self.base_url.clone() - } - - fn captured(&mut self) -> Result { - if let Some(handle) = self.handle.take() { - handle - .join() - .map_err(|_| SwitchyardError::Other("test server thread panicked".into()))??; - } - self.captured - .lock() - .map_err(|_| SwitchyardError::Other("captured mutex poisoned".into()))? - .clone() - .ok_or_else(|| SwitchyardError::Other("test server captured no request".into())) - } -} - -fn serve_once( - listener: TcpListener, - status: u16, - body: Value, - captured: Arc>>, -) -> Result<()> { - let (mut stream, _) = listener - .accept() - .map_err(|error| SwitchyardError::Other(format!("accept failed: {error}")))?; - let mut reader = BufReader::new( - stream - .try_clone() - .map_err(|error| SwitchyardError::Other(format!("clone stream failed: {error}")))?, - ); - let mut first_line = String::new(); - reader - .read_line(&mut first_line) - .map_err(|error| SwitchyardError::Other(format!("read request line failed: {error}")))?; - let path = first_line - .split_whitespace() - .nth(1) - .ok_or_else(|| SwitchyardError::Other("missing request path".into()))? - .to_string(); - { - let mut captured = captured - .lock() - .map_err(|_| SwitchyardError::Other("captured mutex poisoned".into()))?; - *captured = Some(CapturedRequest { path }); - } - - loop { - let mut line = String::new(); - let bytes = reader.read_line(&mut line).map_err(|error| { - SwitchyardError::Other(format!("read request header failed: {error}")) - })?; - if bytes == 0 || line == "\r\n" { - break; - } - } - - let body = serde_json::to_string(&body) - .map_err(|error| SwitchyardError::Other(format!("json encode failed: {error}")))?; - let reason = if status == 200 { "OK" } else { "ERROR" }; - let response = format!( - "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))?; - Ok(()) -} - -fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) -} - -fn profile_with_backends( - targets: Vec, - failures: BTreeMap<&'static str, &'static str>, - max_retries: usize, -) -> Result<(LatencyServiceProfile, Arc>>)> { - profile_with_backend_modes(targets, failures, BTreeMap::new(), max_retries) -} - -fn profile_with_backend_modes( - targets: Vec, - failures: BTreeMap<&'static str, &'static str>, - response_modes: BTreeMap<&'static str, ResponseMode>, - max_retries: usize, -) -> Result<(LatencyServiceProfile, Arc>>)> { - let calls = Arc::new(Mutex::new(Vec::new())); - let mut backends = BTreeMap::new(); - let mut health = BTreeMap::new(); - for target in &targets { - health.insert( - target.id.clone(), - EndpointHealth::new(EndpointHealthStatus::Unknown), - ); - let backend = TestBackend { - name: format!("{}-backend", target.id), - calls: Arc::clone(&calls), - error: failures.get(target.id.as_str()).copied(), - response_mode: response_modes - .get(target.id.as_str()) - .copied() - .unwrap_or(ResponseMode::Completion), - }; - backends.insert( - target.id.clone(), - TargetBackend::new(target.clone(), Arc::new(backend)), - ); - } - - let target_ids = targets.iter().map(|target| target.id.clone()).collect(); - let profile = LatencyServiceProfile { - poller: HealthPoller::new("http://latency.test", target_ids, Duration::from_secs(5))?, - backends, - health: RwLock::new(health), - poll_count: AtomicU64::new(0), - initial_refresh_in_flight: AtomicBool::new(false), - max_retries, - stats: StatsAccumulator::new(), - }; - Ok((profile, calls)) -} - -fn observed(calls: &Arc>>) -> Result> { - calls - .lock() - .map(|calls| calls.clone()) - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string())) -} - -fn profile_input(request: ChatRequest) -> ProfileInput { - ProfileInput { - request, - metadata: RequestMetadata::default(), - } -} - -#[test] -fn profile_config_build_rejects_invalid_config() -> Result<()> { - let fast = target("fast", "upstream-fast")?; - let duplicate = LatencyServiceProfileConfig { - latency_service_url: "http://latency.test".to_string(), - targets: vec![fast.clone(), fast], - poll_timeout_secs: 5.0, - max_retries: 2, - }; - match duplicate.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("duplicate target fast")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "duplicate targets should reject profile construction".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - - let empty_url = LatencyServiceProfileConfig { - latency_service_url: " ".to_string(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 2, - }; - match empty_url.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("requires latency_service_url")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "empty URL should reject profile construction".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - - Ok(()) -} - -#[test] -fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { - let config = LatencyServiceProfileConfig { - latency_service_url: "http://latency.test".to_string(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 2, - }; - assert_eq!(LatencyServiceProfileConfig::PROFILE_TYPE, "latency-service"); - assert_eq!(config.profile_type(), "latency-service"); - - let unknown_field = json!({ - "latency_service_url": "http://latency.test", - "targets": config.targets, - "poll_timeout_secs": 5.0, - "max_retries": 2, - "poll_interval_secs": 1.0, - }); - let error = serde_json::from_value::(unknown_field) - .err() - .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?; - assert!(error.to_string().contains("unknown field")); - Ok(()) -} - -#[tokio::test] -async fn healthy_endpoint_serves_request_and_records_stats() -> Result<()> { - let (profile, calls) = profile_with_backends( - vec![ - target("fast", "upstream-fast")?, - target("slow", "upstream-slow")?, - ], - BTreeMap::new(), - 0, - )?; - profile.update_health( - LlmTargetId::from_static("fast"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0), - )?; - profile.update_health( - LlmTargetId::from_static("slow"), - EndpointHealth::with_latency(EndpointHealthStatus::Degraded, 10.0), - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "incoming-model", - "messages": [{"role": "user", "content": "hi"}], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("upstream-fast") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy")); - assert_eq!( - routing_metadata.router_version.as_deref(), - Some("latency-service:v1") - ); - let response = response.response; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "fast-backend"); - assert_eq!(calls[0].body["model"], "upstream-fast"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["backend"], "fast-backend"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - - let snapshot = profile.stats.snapshot()?; - let model = snapshot - .models - .get("upstream-fast") - .ok_or_else(|| SwitchyardError::Other("selected model stats should exist".into()))?; - assert_eq!(model.calls, 1); - assert_eq!(snapshot.total_tokens.prompt, 13); - assert_eq!(snapshot.total_tokens.completion, 5); - Ok(()) -} - -#[tokio::test] -async fn retry_avoids_failed_target_while_alternative_exists() -> Result<()> { - let (profile, calls) = profile_with_backends( - vec![ - target("failing", "upstream-failing")?, - target("fallback", "upstream-fallback")?, - ], - BTreeMap::from([("failing", "down")]), - 1, - )?; - profile.update_health( - LlmTargetId::from_static("failing"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - )?; - profile.update_health( - LlmTargetId::from_static("fallback"), - EndpointHealth::new(EndpointHealthStatus::Degraded), - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "incoming-model", - "messages": [], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("upstream-fallback") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("degraded")); - let response = response.response; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].backend, "failing-backend"); - assert_eq!(calls[1].backend, "fallback-backend"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["backend"], "fallback-backend"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - Ok(()) -} - -#[tokio::test] -async fn exhausted_retries_return_last_target_error() -> Result<()> { - let (profile, calls) = profile_with_backends( - vec![ - target("first", "upstream-first")?, - target("second", "upstream-second")?, - ], - BTreeMap::from([("first", "first down"), ("second", "second down")]), - 5, - )?; - profile.update_health( - LlmTargetId::from_static("first"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - )?; - profile.update_health( - LlmTargetId::from_static("second"), - EndpointHealth::new(EndpointHealthStatus::Degraded), - )?; - - let error = match profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "incoming-model", - "messages": [], - })))) - .await - { - Ok(_) => { - return Err(SwitchyardError::Other( - "all failing targets should return an error".into(), - )); - } - Err(error) => error, - }; - - assert!( - error.to_string().contains("second down"), - "last target error should be returned, got {error:?}" - ); - let calls = observed(&calls)?; - assert_eq!( - calls - .iter() - .map(|call| call.backend.as_str()) - .collect::>(), - vec!["first-backend", "second-backend"] - ); - Ok(()) -} - -#[tokio::test] -async fn process_only_rewrites_request_and_does_not_call_backend() -> Result<()> { - let (profile, calls) = profile_with_backends( - vec![ - target("fast", "upstream-fast")?, - target("slow", "upstream-slow")?, - ], - BTreeMap::new(), - 0, - )?; - profile.update_health( - LlmTargetId::from_static("fast"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - )?; - - let request = profile - .process(profile_input(ChatRequest::openai_chat(json!({ - "model": "incoming-model", - "messages": [], - })))) - .await?; - - assert_eq!(request.profile_input.request.model(), Some("upstream-fast")); - assert_eq!(request.selected.target_id, LlmTargetId::from_static("fast")); - assert!(observed(&calls)?.is_empty()); - Ok(()) -} - -#[tokio::test] -async fn streaming_response_is_passed_through_from_selected_target() -> Result<()> { - let (profile, calls) = profile_with_backend_modes( - vec![target("fast", "upstream-fast")?], - BTreeMap::new(), - BTreeMap::from([("fast", ResponseMode::OpenAiStream)]), - 0, - )?; - profile.update_health( - LlmTargetId::from_static("fast"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "incoming-model", - "messages": [], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("upstream-fast") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy")); - let response = response.response; - - match response { - ChatResponse::OpenAiStream(_) => {} - _ => { - return Err(SwitchyardError::Other( - "expected OpenAI stream response".into(), - )) - } - } - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "fast-backend"); - assert_eq!(calls[0].body["model"], "upstream-fast"); - Ok(()) -} - -#[test] -fn unknown_target_health_update_is_rejected() -> Result<()> { - let (profile, _calls) = - profile_with_backends(vec![target("known", "upstream-known")?], BTreeMap::new(), 0)?; - - let error = match profile.update_health( - LlmTargetId::from_static("missing"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - ) { - Ok(_) => { - return Err(SwitchyardError::Other( - "unknown target health update should fail".into(), - )); - } - Err(error) => error, - }; - - assert!(error - .to_string() - .contains("target missing is not configured")); - Ok(()) -} - -#[tokio::test] -async fn poll_once_updates_cache_and_ignores_unknown_service_ids() -> Result<()> { - let mut server = OneShotServer::json( - 200, - json!({ - "endpoint_health": { - "fast": {"status": "healthy", "last_latency_ms": 42.0}, - "unknown-service-id": {"status": "degraded", "last_latency_ms": 999.0} - } - }), - )?; - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - - profile.poll_once().await?; - let captured = server.captured()?; - - assert!(captured.path.starts_with("/v1/endpoints/health?")); - assert!(captured.path.contains("endpoint_ids=fast")); - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("fast")), - Some(&EndpointHealth::with_latency( - EndpointHealthStatus::Healthy, - 42.0 - )) - ); - assert!(profile.is_ready()); - Ok(()) -} - -#[tokio::test] -async fn poll_once_resets_omitted_known_targets_to_unknown() -> Result<()> { - let mut server = OneShotServer::json( - 200, - json!({ - "endpoint_health": { - "fast": {"status": "healthy", "last_latency_ms": 42.0} - } - }), - )?; - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![ - target("fast", "upstream-fast")?, - target("slow", "upstream-slow")?, - ], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - profile.update_health( - LlmTargetId::from_static("slow"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 7.0), - )?; - - profile.poll_once().await?; - let _captured = server.captured()?; - - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("fast")), - Some(&EndpointHealth::with_latency( - EndpointHealthStatus::Healthy, - 42.0 - )) - ); - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("slow")), - Some(&EndpointHealth::new(EndpointHealthStatus::Unknown)) - ); - Ok(()) -} - -#[tokio::test] -async fn poll_once_tolerates_additive_health_payload_fields() -> Result<()> { - let mut server = OneShotServer::json( - 200, - json!({ - "endpoint_health": { - "fast": { - "status": "healthy", - "last_latency_ms": 42.0, - "reason": "warm" - } - }, - "generated_at": "2026-05-19T00:00:00Z" - }), - )?; - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - - profile.poll_once().await?; - let _captured = server.captured()?; - - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("fast")), - Some(&EndpointHealth::with_latency( - EndpointHealthStatus::Healthy, - 42.0 - )) - ); - Ok(()) -} - -#[tokio::test] -async fn poll_failure_resets_stale_health_to_unknown_without_marking_ready() -> Result<()> { - let mut server = OneShotServer::json(500, json!({"error": "down"}))?; - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - profile.update_health( - LlmTargetId::from_static("fast"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 42.0), - )?; - - let error = match profile.poll_once().await { - Ok(_) => return Err(SwitchyardError::Other("poll should fail".into())), - Err(error) => error, - }; - - assert!(error.to_string().contains("HTTP 500")); - let _captured = server.captured()?; - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("fast")), - Some(&EndpointHealth::new(EndpointHealthStatus::Unknown)) - ); - assert!(!profile.is_ready()); - Ok(()) -} - -#[tokio::test] -async fn malformed_health_status_resets_to_unknown() -> Result<()> { - let mut server = OneShotServer::json( - 200, - json!({ - "endpoint_health": { - "fast": {"status": "on_fire", "last_latency_ms": 42.0} - } - }), - )?; - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![target("fast", "upstream-fast")?], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - profile.update_health( - LlmTargetId::from_static("fast"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 42.0), - )?; - - let error = match profile.poll_once().await { - Ok(_) => { - return Err(SwitchyardError::Other( - "malformed health status should fail".into(), - )); - } - Err(error) => error, - }; - - assert!( - error.to_string().contains("invalid JSON"), - "malformed status should surface as invalid response, got {error:?}" - ); - let _captured = server.captured()?; - assert_eq!( - profile - .health_snapshot() - .get(&LlmTargetId::from_static("fast")), - Some(&EndpointHealth::new(EndpointHealthStatus::Unknown)) - ); - Ok(()) -} - -#[test] -fn faster_endpoint_gets_more_traffic_within_same_tier() -> Result<()> { - let snapshot = BTreeMap::from([ - ( - LlmTargetId::from_static("fast"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0), - ), - ( - LlmTargetId::from_static("slow"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 500.0), - ), - ]); - - let mut fast = 0; - let mut slow = 0; - for _ in 0..2_000 { - match select_target(&snapshot, &BTreeSet::new())? - .target_id - .as_str() - { - "fast" => fast += 1, - "slow" => slow += 1, - other => { - return Err(SwitchyardError::Other(format!( - "unexpected target selected: {other}" - ))); - } - } - } - - assert!( - fast > slow * 5, - "expected fast endpoint to dominate inverse-latency routing; fast={fast}, slow={slow}" - ); - Ok(()) -} - -#[test] -fn missing_latency_falls_back_to_uniform_random() -> Result<()> { - let snapshot = BTreeMap::from([ - ( - LlmTargetId::from_static("with-latency"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0), - ), - ( - LlmTargetId::from_static("without-latency"), - EndpointHealth::new(EndpointHealthStatus::Healthy), - ), - ]); - - let mut first = 0; - let mut second = 0; - for _ in 0..2_000 { - match select_target(&snapshot, &BTreeSet::new())? - .target_id - .as_str() - { - "with-latency" => first += 1, - "without-latency" => second += 1, - other => { - return Err(SwitchyardError::Other(format!( - "unexpected target selected: {other}" - ))); - } - } - } - - assert!( - (850..=1150).contains(&first), - "uniform fallback should stay near 50/50; first={first}, second={second}" - ); - assert!( - (850..=1150).contains(&second), - "uniform fallback should stay near 50/50; first={first}, second={second}" - ); - Ok(()) -} - -#[test] -fn non_positive_latency_selects_without_error_or_dividing_by_zero() -> Result<()> { - let snapshot = BTreeMap::from([ - ( - LlmTargetId::from_static("zero"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 0.0), - ), - ( - LlmTargetId::from_static("positive"), - EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 100.0), - ), - ]); - - for _ in 0..100 { - let selected = select_target(&snapshot, &BTreeSet::new())?; - assert!( - selected.target_id == LlmTargetId::from_static("zero") - || selected.target_id == LlmTargetId::from_static("positive") - ); - } - Ok(()) -} diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing.rs b/crates/switchyard-components-v2/src/profiles/llm_routing.rs deleted file mode 100644 index f44ca517..00000000 --- a/crates/switchyard-components-v2/src/profiles/llm_routing.rs +++ /dev/null @@ -1,1602 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! LLM classifier routing as a profile-owned runtime. - -use std::collections::BTreeSet; -use std::time::Instant; - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; -use switchyard_components::stats::usage_from_body; -use switchyard_components::StatsAccumulator; -use switchyard_core::{ - BackendFormat, ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError, -}; - -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::stats_recording::record_usage_or_wrap_stream; -use crate::{ - profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse, - RoutingMetadata, -}; - -const TIER_STRONG: &str = "strong"; -const TIER_WEAK: &str = "weak"; -const PROFILE_GENERAL: &str = "general"; -const PROFILE_CODING_AGENT: &str = "coding_agent"; -const PROFILE_OPENCLAW: &str = "openclaw"; -const DEFAULT_CLASSIFIER_MAX_TOKENS: u64 = 4096; -const DEFAULT_ALIGNMENT_MIN_CONFIDENCE: f64 = 0.85; -const DEFAULT_CLASSIFIER_TOOL_NAME: &str = "select_route"; - -/// Config for the flatter LLM classifier profile. -#[profile_config("llm-routing")] -pub struct LlmRoutingProfileConfig { - /// Strong target served by this profile. - #[profile_target] - pub strong: LlmTarget, - /// Weak target served by this profile. - #[profile_target] - pub weak: LlmTarget, - /// OpenAI-compatible classifier target used for tool-call route selection. - #[profile_target] - pub classifier: LlmTarget, - /// Target used for one retry after context-window overflow. - #[serde(default = "default_fallback_target_on_evict")] - pub fallback_target_on_evict: LlmTargetId, - /// Classifier policy profile: `general`, `coding_agent`, or `openclaw`. - #[serde(default = "default_profile_name")] - pub profile_name: String, - /// Confidence floor below which routing falls back to the profile default. - #[serde(default)] - pub classifier_min_confidence: f64, - /// Whether classifier failures fall open to the default tier. - #[serde(default = "default_classifier_fail_open")] - pub classifier_fail_open: bool, - /// Number of recent turns included in the classifier request summary. - #[serde(default = "default_recent_turn_window")] - pub classifier_recent_turn_window: usize, - /// Maximum tokens allowed for the classifier tool-call response. - #[serde(default = "default_classifier_max_tokens")] - pub classifier_max_tokens: u64, - /// Confidence required before the classifier recommendation can bump the policy tier. - #[serde(default = "default_alignment_min_confidence")] - pub alignment_min_confidence: f64, - /// Default backend tier used for abstain, low-confidence, and fail-open decisions. - #[serde(default)] - pub default_tier: Option, - /// Optional mapping from classifier route tiers to backend tiers. - #[serde(default)] - pub tier_mapping: Option, - /// Optional system prompt override for the classifier request. - #[serde(default)] - pub classifier_system_prompt: Option, - /// Optional function/tool name override for classifier route selection. - #[serde(default)] - pub classifier_tool_name: Option, - /// Optional function/tool description override for classifier route selection. - #[serde(default)] - pub classifier_tool_description: Option, - /// Optional JSON schema override for classifier tool arguments. - #[serde(default)] - pub classifier_tool_parameters: Option, -} - -/// Mapping from classifier route tiers to configured backend tiers. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct LlmRoutingTierMapping { - /// Backend tier for `recommended_tier: simple`. - pub simple: String, - /// Backend tier for `recommended_tier: medium`. - pub medium: String, - /// Backend tier for `recommended_tier: complex`. - pub complex: String, - /// Backend tier for `recommended_tier: reasoning`. - pub reasoning: String, -} - -impl ProfileConfig for LlmRoutingProfileConfig { - type Runtime = LlmRoutingProfile; - - /// Builds the runtime profile using profile-local targets and backends. - fn build(&self) -> Result { - validate_config(self)?; - let policy = ClassifierPolicy::from_name(&self.profile_name)?; - let strong_target = llm_routing_target(self.strong.clone()); - let weak_target = llm_routing_target(self.weak.clone()); - let classifier_target = llm_routing_classifier_target(self.classifier.clone()); - Ok(LlmRoutingProfile { - policy, - strong_backend: native_target_backend(strong_target)?, - weak_backend: native_target_backend(weak_target)?, - classifier_backend: native_target_backend(classifier_target.clone())?, - classifier_target, - fallback_target_on_evict: self.fallback_target_on_evict.clone(), - classifier_min_confidence: self.classifier_min_confidence, - classifier_fail_open: self.classifier_fail_open, - classifier_recent_turn_window: self.classifier_recent_turn_window, - classifier_max_tokens: self.classifier_max_tokens, - alignment_min_confidence: self.alignment_min_confidence, - default_tier: resolve_default_tier(self.default_tier.as_deref())?, - tier_mapping: self - .tier_mapping - .clone() - .unwrap_or_else(|| policy.default_tier_mapping()), - classifier_tool: LlmRoutingClassifierTool::from_config(policy, self), - stats: profile_stats_accumulator(), - }) - } -} - -/// LLM classifier profile runtime. -pub struct LlmRoutingProfile { - policy: ClassifierPolicy, - strong_backend: TargetBackend, - weak_backend: TargetBackend, - classifier_backend: TargetBackend, - classifier_target: LlmTarget, - fallback_target_on_evict: LlmTargetId, - classifier_min_confidence: f64, - classifier_fail_open: bool, - classifier_recent_turn_window: usize, - classifier_max_tokens: u64, - alignment_min_confidence: f64, - default_tier: String, - tier_mapping: LlmRoutingTierMapping, - classifier_tool: LlmRoutingClassifierTool, - stats: StatsAccumulator, -} - -/// Processed LLM-routing request with the profile-owned route decision. -pub struct LlmRoutingProcessedRequest { - /// Routed input prepared for the selected backend. - pub profile_input: ProfileInput, - /// Selected LLM-routing decision for this request. - pub decision: LlmRoutingDecision, -} - -/// Serializable decision record used by Rust callers and tests. -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct LlmRoutingDecision { - /// Concrete tier label selected for backend dispatch. - pub tier: String, - /// Decision source (`policy_tier`, `low_confidence`, `abstain`, etc.). - pub source: String, - /// Human-readable reason for the tier selection. - pub reason: String, - /// Policy tier computed from classifier features. - pub policy_tier: Option, - /// Classifier-emitted recommended tier. - pub llm_recommended_tier: Option, - /// Classifier confidence, when a classifier output was available. - pub confidence: Option, - /// Selected v2 target id. - pub selected_target: String, - /// Selected upstream model. - pub selected_model: String, - /// Raw classifier signals as JSON for audit/debugging. - pub signals: Option, -} - -impl LlmRoutingProfile { - async fn classify(&self, request: &ChatRequest) -> Result { - let classifier_request = ChatRequest::openai_chat(json!({ - "model": self.classifier_target.model.as_str(), - "messages": [ - {"role": "system", "content": self.classifier_tool.system_prompt.as_str()}, - {"role": "user", "content": summarize_request( - request, - self.classifier_recent_turn_window, - )}, - ], - "temperature": 0, - "max_tokens": self.classifier_max_tokens, - "tools": [self.classifier_tool.definition()], - "tool_choice": self.classifier_tool.choice(), - })); - - let started_at = Instant::now(); - let response = self.classifier_backend.call(&classifier_request).await?; - let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0; - let raw = classifier_tool_arguments(&response, &self.classifier_tool.name)?; - let signals = self.policy.parse_signals(&raw)?; - validate_signals(&signals)?; - let usage = response.body().map(usage_from_body).unwrap_or_default(); - self.stats.record_classifier_usage( - self.classifier_target.model.as_str(), - usage, - Some(latency_ms), - )?; - Ok(signals) - } - - async fn route_request(&self, mut input: ProfileInput) -> Result { - normalize_reasoning_effort(&mut input.request); - let decision = match self.classify(&input.request).await { - Ok(signals) => self.select(&signals)?, - Err(error) => { - self.stats - .record_classifier_error(self.classifier_target.model.as_str())?; - if self.classifier_fail_open { - let signals = ClassifierSignals::abstain(self.policy); - self.default_decision( - "classifier_error_fall_open", - "classifier failed and classifier_fail_open routed to the default tier", - &signals, - )? - } else { - return Err(classifier_error(error)); - } - } - }; - input.request.set_model(&decision.selected_model); - Ok(LlmRoutingProcessedRequest { - profile_input: input, - decision, - }) - } - - fn select(&self, signals: &ClassifierSignals) -> Result { - let selected = if signals.abstain { - self.default_decision("abstain", "classifier abstained", signals)? - } else if signals.confidence < self.classifier_min_confidence { - self.default_decision( - "low_confidence", - &format!( - "classifier confidence {:.3} < min_confidence {:.3}", - signals.confidence, self.classifier_min_confidence - ), - signals, - )? - } else { - let mut policy_tier = self.policy.policy_tier(signals); - let mut source = "policy_tier"; - let mut reason = format!("policy_tier() returned {:?}", policy_tier.as_str()); - if self.policy.align_with_llm_recommendation() - && signals.confidence >= self.alignment_min_confidence - && matches!(policy_tier, RouteTier::Simple | RouteTier::Medium) - && matches!( - signals.recommended_tier, - RouteTier::Complex | RouteTier::Reasoning - ) - { - policy_tier = signals.recommended_tier; - source = "llm_alignment_bump"; - reason = format!( - "LLM alignment bump to {:?} at confidence {:.3}", - policy_tier.as_str(), - signals.confidence - ); - } - let mut tier = self.tier_for(policy_tier); - if self.policy.escalate_on_tool_planning() - && tier != self.default_tier - && self.policy.requires_tool_planning(signals) - { - tier = self.default_tier.clone(); - source = "tool_planning_escalation"; - reason = format!( - "tool planning escalation from {:?} to {:?}", - self.tier_for(policy_tier), - tier - ); - } - self.decision(LlmRoutingDecisionInput { - tier, - source, - reason, - policy_tier: Some(policy_tier), - llm_recommended_tier: Some(signals.recommended_tier), - confidence: Some(signals.confidence), - signals: Some(signals.raw.clone()), - })? - }; - Ok(selected) - } - - fn default_decision( - &self, - source: &'static str, - reason: &str, - signals: &ClassifierSignals, - ) -> Result { - self.decision(LlmRoutingDecisionInput { - tier: self.default_tier.clone(), - source, - reason: reason.to_string(), - policy_tier: Some(self.policy.policy_tier(signals)), - llm_recommended_tier: Some(signals.recommended_tier), - confidence: Some(signals.confidence), - signals: Some(signals.raw.clone()), - }) - } - - fn decision(&self, input: LlmRoutingDecisionInput) -> Result { - let backend = self.backend_for_tier(&input.tier)?; - Ok(LlmRoutingDecision { - tier: input.tier, - source: input.source.to_string(), - reason: input.reason, - policy_tier: input.policy_tier.map(|tier| tier.as_str().to_string()), - llm_recommended_tier: input - .llm_recommended_tier - .map(|tier| tier.as_str().to_string()), - confidence: input.confidence, - selected_target: backend.target().id.to_string(), - selected_model: backend.target().model.to_string(), - signals: input.signals, - }) - } - - fn backend_for_tier(&self, tier: &str) -> Result<&TargetBackend> { - match tier { - TIER_STRONG => Ok(&self.strong_backend), - TIER_WEAK => Ok(&self.weak_backend), - other => Err(SwitchyardError::InvalidConfig(format!( - "llm-routing selected unknown tier {other:?}" - ))), - } - } - - fn tier_for(&self, tier: RouteTier) -> String { - self.tier_mapping.tier_for(tier).to_string() - } - - fn backend_for_target(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> { - if *target_id == self.strong_backend.target().id { - Ok(&self.strong_backend) - } else if *target_id == self.weak_backend.target().id { - Ok(&self.weak_backend) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "llm-routing selected target {target_id} that is not configured for this profile" - ))) - } - } - - fn tier_for_target(&self, target_id: &LlmTargetId) -> Result<&'static str> { - if *target_id == self.strong_backend.target().id { - Ok(TIER_STRONG) - } else if *target_id == self.weak_backend.target().id { - Ok(TIER_WEAK) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "llm-routing target {target_id} is not configured for this profile" - ))) - } - } - - fn fallback_processed_request( - &self, - processed: &LlmRoutingProcessedRequest, - ) -> Result { - let backend = self.backend_for_target(&self.fallback_target_on_evict)?; - let target = backend.target(); - let mut profile_input = processed.profile_input.clone(); - profile_input.request.set_model(target.model.as_str()); - let mut decision = processed.decision.clone(); - decision.tier = self.tier_for_target(&target.id)?.to_string(); - decision.source = "context_overflow_fallback".to_string(); - decision.reason = format!( - "selected target {} exceeded its context window; retried fallback target {}", - processed.decision.selected_target, target.id - ); - decision.selected_target = target.id.to_string(); - decision.selected_model = target.model.to_string(); - Ok(LlmRoutingProcessedRequest { - profile_input, - decision, - }) - } - - async fn call_selected( - &self, - processed: &LlmRoutingProcessedRequest, - ) -> (Result, f64) { - let started_at = Instant::now(); - let backend = match self.backend_for_target_id_str(&processed.decision.selected_target) { - Ok(backend) => backend, - Err(error) => return (Err(error), 0.0), - }; - let result = backend.call(&processed.profile_input.request).await; - let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0; - (result, latency_ms) - } - - fn backend_for_target_id_str(&self, target_id: &str) -> Result<&TargetBackend> { - if target_id == self.strong_backend.target().id.as_str() { - Ok(&self.strong_backend) - } else if target_id == self.weak_backend.target().id.as_str() { - Ok(&self.weak_backend) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "llm-routing selected target {target_id:?} that is not configured for this profile" - ))) - } - } - - fn record_success( - &self, - decision: &LlmRoutingDecision, - response: ChatResponse, - profile_started_at: Instant, - backend_latency_ms: f64, - ) -> Result { - self.stats.record_success( - decision.selected_model.as_str(), - Some(backend_latency_ms), - Some(decision.tier.as_str()), - )?; - record_usage_or_wrap_stream( - &self.stats, - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - profile_started_at, - backend_latency_ms, - response, - ) - } - - fn record_error(&self, decision: &LlmRoutingDecision) -> Result<()> { - self.stats.record_error( - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - ) - } - - fn routing_metadata(&self, decision: &LlmRoutingDecision) -> RoutingMetadata { - RoutingMetadata { - selected_model: Some(decision.selected_model.clone()), - selected_tier: Some(decision.tier.clone()), - confidence: decision.confidence, - router_version: Some(format!("llm-routing:{}:v1", self.policy.as_str())), - tolerance: Some(self.classifier_min_confidence), - rationale: Some(decision.reason.clone()), - } - } -} - -#[async_trait] -impl ProfileHooks for LlmRoutingProfile { - type ProcessedRequest = LlmRoutingProcessedRequest; - - /// Runs classifier routing and returns a prepared backend request. - async fn process(&self, input: ProfileInput) -> Result { - self.route_request(input).await - } - - /// Leaves the backend response unchanged after LLM routing completes. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for LlmRoutingProfile { - /// Executes LLM classifier routing with one context-window fallback retry. - async fn run(&self, input: ProfileInput) -> Result { - let profile_started_at = Instant::now(); - let processed = self.process(input).await?; - let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await; - match first_result { - Ok(response) => { - let response = self.record_success( - &processed.decision, - response, - profile_started_at, - first_backend_latency_ms, - )?; - let response = self.rprocess(&processed, response).await?; - return Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(&processed.decision), - )); - } - Err(SwitchyardError::ContextWindowExceeded { .. }) => { - let retry = self.fallback_processed_request(&processed)?; - let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await; - match retry_result { - Ok(response) => { - let response = self.record_success( - &retry.decision, - response, - profile_started_at, - retry_backend_latency_ms, - )?; - let response = self.rprocess(&retry, response).await?; - return Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(&retry.decision), - )); - } - Err(SwitchyardError::ContextWindowExceeded { target_id, .. }) => { - self.record_error(&retry.decision)?; - return Err(SwitchyardError::ContextPoolExhausted { - last_target_id: target_id, - reason: "all attempted targets returned context-window overflow" - .to_string(), - }); - } - Err(error) => { - self.record_error(&retry.decision)?; - return Err(error); - } - } - } - Err(error) => { - self.record_error(&processed.decision)?; - return Err(error); - } - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum RouteTier { - Simple, - Medium, - Complex, - Reasoning, -} - -struct LlmRoutingDecisionInput { - tier: String, - source: &'static str, - reason: String, - policy_tier: Option, - llm_recommended_tier: Option, - confidence: Option, - signals: Option, -} - -impl RouteTier { - fn parse(raw: &str) -> Option { - match raw { - "simple" => Some(Self::Simple), - "medium" => Some(Self::Medium), - "complex" => Some(Self::Complex), - "reasoning" => Some(Self::Reasoning), - _ => None, - } - } - - fn as_str(self) -> &'static str { - match self { - Self::Simple => "simple", - Self::Medium => "medium", - Self::Complex => "complex", - Self::Reasoning => "reasoning", - } - } -} - -#[derive(Clone, Debug)] -struct LlmRoutingClassifierTool { - name: String, - description: String, - parameters: Value, - system_prompt: String, -} - -impl LlmRoutingClassifierTool { - fn from_config(policy: ClassifierPolicy, config: &LlmRoutingProfileConfig) -> Self { - Self { - name: config - .classifier_tool_name - .clone() - .unwrap_or_else(default_classifier_tool_name), - description: config - .classifier_tool_description - .clone() - .unwrap_or_else(|| policy.default_tool_description().to_string()), - parameters: config - .classifier_tool_parameters - .clone() - .unwrap_or_else(|| policy.schema()), - system_prompt: config - .classifier_system_prompt - .clone() - .unwrap_or_else(|| policy.default_system_prompt().to_string()), - } - } - - fn definition(&self) -> Value { - json!({ - "type": "function", - "function": { - "name": self.name.as_str(), - "description": self.description.as_str(), - "parameters": self.parameters.clone(), - "strict": true, - }, - }) - } - - fn choice(&self) -> Value { - json!({ - "type": "function", - "function": {"name": self.name.as_str()}, - }) - } -} - -impl LlmRoutingTierMapping { - fn tier_for(&self, tier: RouteTier) -> &str { - match tier { - RouteTier::Simple => &self.simple, - RouteTier::Medium => &self.medium, - RouteTier::Complex => &self.complex, - RouteTier::Reasoning => &self.reasoning, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ClassifierPolicy { - General, - CodingAgent, - OpenClaw, -} - -impl ClassifierPolicy { - fn from_name(name: &str) -> Result { - match name { - PROFILE_GENERAL => Ok(Self::General), - PROFILE_CODING_AGENT => Ok(Self::CodingAgent), - PROFILE_OPENCLAW => Ok(Self::OpenClaw), - other => Err(SwitchyardError::InvalidConfig(format!( - "unknown LLM classifier profile {other:?}; expected general, coding_agent, or openclaw" - ))), - } - } - - fn as_str(self) -> &'static str { - match self { - Self::General => PROFILE_GENERAL, - Self::CodingAgent => PROFILE_CODING_AGENT, - Self::OpenClaw => PROFILE_OPENCLAW, - } - } - - fn default_recommended_tier(self) -> RouteTier { - RouteTier::Medium - } - - fn default_tier_mapping(self) -> LlmRoutingTierMapping { - match self { - Self::General => LlmRoutingTierMapping { - simple: TIER_WEAK.to_string(), - medium: TIER_STRONG.to_string(), - complex: TIER_STRONG.to_string(), - reasoning: TIER_STRONG.to_string(), - }, - Self::CodingAgent | Self::OpenClaw => LlmRoutingTierMapping { - simple: TIER_WEAK.to_string(), - medium: TIER_WEAK.to_string(), - complex: TIER_STRONG.to_string(), - reasoning: TIER_STRONG.to_string(), - }, - } - } - - fn escalate_on_tool_planning(self) -> bool { - matches!(self, Self::CodingAgent | Self::OpenClaw) - } - - fn align_with_llm_recommendation(self) -> bool { - matches!(self, Self::CodingAgent | Self::OpenClaw) - } - - fn default_system_prompt(self) -> &'static str { - match self { - Self::General => include_str!("llm_routing/prompts/general.md"), - Self::CodingAgent => include_str!("llm_routing/prompts/coding_agent.md"), - Self::OpenClaw => include_str!("llm_routing/prompts/openclaw.md"), - } - } - - fn default_tool_description(self) -> &'static str { - match self { - Self::General => "Select the backend tier for a general LLM request.", - Self::CodingAgent => "Select the backend tier for a coding-agent request.", - Self::OpenClaw => "Select the backend tier for an OpenClaw assistant request.", - } - } - - fn schema(self) -> Value { - match self { - Self::General => general_schema(), - Self::CodingAgent => coding_agent_schema(), - Self::OpenClaw => openclaw_schema(), - } - } - - fn parse_signals(self, raw: &str) -> Result { - let value: Value = serde_json::from_str(raw).map_err(|error| { - SwitchyardError::Backend(format!("classifier returned invalid JSON: {error}")) - })?; - ClassifierSignals::from_value(self, value) - } - - fn policy_tier(self, signals: &ClassifierSignals) -> RouteTier { - match self { - Self::General => general_policy_tier(signals), - Self::CodingAgent => coding_agent_policy_tier(signals), - Self::OpenClaw => openclaw_policy_tier(signals), - } - } - - fn requires_tool_planning(self, signals: &ClassifierSignals) -> bool { - match self { - Self::General => bool_field(&signals.raw, "tool_planning_required"), - Self::CodingAgent => { - let turn_type = str_field(&signals.raw, "turn_type"); - let scope = str_field(&signals.raw, "code_modification_scope"); - let tool_count = u64_field(&signals.raw, "tool_call_count_estimate"); - let scope_is_modifying = matches!( - scope, - Some("function" | "file" | "multi_file" | "cross_module") - ); - turn_type == Some("planning") || (tool_count >= 3 && scope_is_modifying) - } - Self::OpenClaw => { - u64_field(&signals.raw, "tool_call_count_estimate") >= 2 - || str_field(&signals.raw, "turn_type") == Some("tool_orchestration") - } - } - } -} - -#[derive(Clone, Debug, PartialEq)] -struct ClassifierSignals { - raw: Value, - recommended_tier: RouteTier, - confidence: f64, - abstain: bool, -} - -impl ClassifierSignals { - fn from_value(policy: ClassifierPolicy, raw: Value) -> Result { - let recommended_tier = str_field(&raw, "recommended_tier") - .and_then(RouteTier::parse) - .ok_or_else(|| { - SwitchyardError::Backend( - "classifier JSON missing valid recommended_tier".to_string(), - ) - })?; - let confidence = raw - .get("confidence") - .and_then(Value::as_f64) - .ok_or_else(|| { - SwitchyardError::Backend("classifier JSON missing numeric confidence".to_string()) - })?; - let abstain = raw.get("abstain").and_then(Value::as_bool).unwrap_or(false); - let signals = Self { - raw, - recommended_tier, - confidence, - abstain, - }; - validate_policy_fields(policy, &signals)?; - Ok(signals) - } - - fn abstain(policy: ClassifierPolicy) -> Self { - let recommended_tier = policy.default_recommended_tier(); - let base = json!({ - "recommended_tier": recommended_tier.as_str(), - "confidence": 0.0, - "abstain": true, - }); - let raw = match policy { - ClassifierPolicy::General => merge_json_objects( - base, - json!({ - "task_type": "other", - "complexity": "medium", - "reasoning_depth": "light", - "tool_planning_required": false, - "precision_requirement": "medium", - "context_dependency": "conversation", - "structured_output_risk": "medium", - "reason_code": "ambiguous", - }), - ), - ClassifierPolicy::CodingAgent => merge_json_objects( - base, - json!({ - "turn_type": "exploration", - "code_modification_scope": "none", - "tool_call_count_estimate": 0, - "requires_codebase_context": false, - }), - ), - ClassifierPolicy::OpenClaw => merge_json_objects( - base, - json!({ - "turn_type": "chitchat", - "tool_call_count_estimate": 0, - "memory_dependency": "none", - "external_action_required": false, - "precision_requirement": "low", - "ambiguity": "low", - "channel_kind": "casual", - }), - ), - }; - Self { - raw, - recommended_tier, - confidence: 0.0, - abstain: true, - } - } -} - -fn merge_json_objects(mut left: Value, right: Value) -> Value { - let (Some(left), Some(right)) = (left.as_object_mut(), right.as_object()) else { - return left; - }; - for (key, value) in right { - left.insert(key.clone(), value.clone()); - } - Value::Object(left.clone()) -} - -fn validate_signals(signals: &ClassifierSignals) -> Result<()> { - if signals.confidence.is_finite() && (0.0..=1.0).contains(&signals.confidence) { - return Ok(()); - } - Err(SwitchyardError::Backend(format!( - "classifier confidence must be finite and in [0.0, 1.0], got {:?}", - signals.confidence - ))) -} - -fn validate_policy_fields(policy: ClassifierPolicy, signals: &ClassifierSignals) -> Result<()> { - let required = match policy { - ClassifierPolicy::General => &[ - "task_type", - "complexity", - "reasoning_depth", - "tool_planning_required", - "precision_requirement", - "context_dependency", - "structured_output_risk", - "reason_code", - ][..], - ClassifierPolicy::CodingAgent => &[ - "turn_type", - "code_modification_scope", - "tool_call_count_estimate", - "requires_codebase_context", - ][..], - ClassifierPolicy::OpenClaw => &[ - "turn_type", - "tool_call_count_estimate", - "memory_dependency", - "external_action_required", - "precision_requirement", - "ambiguity", - "channel_kind", - ][..], - }; - for field in required { - if signals.raw.get(*field).is_none() { - return Err(SwitchyardError::Backend(format!( - "classifier JSON missing required field {field:?}" - ))); - } - } - Ok(()) -} - -fn general_policy_tier(signals: &ClassifierSignals) -> RouteTier { - let mut scores = [0_i32; 4]; - match str_field(&signals.raw, "complexity") { - Some("simple") => scores[0] += 2, - Some("medium") => scores[1] += 2, - Some("complex") => scores[2] += 2, - Some("reasoning") => scores[3] += 2, - _ => {} - } - match str_field(&signals.raw, "reasoning_depth") { - Some("deep") => scores[3] += 2, - Some("multi_step") => scores[2] += 1, - _ => {} - } - if bool_field(&signals.raw, "tool_planning_required") { - scores[2] += 1; - } - if str_field(&signals.raw, "precision_requirement") == Some("high") { - scores[2] += 1; - } - if str_field(&signals.raw, "structured_output_risk") == Some("high") { - scores[2] += 1; - } - argmax_tier(scores) -} - -fn coding_agent_policy_tier(signals: &ClassifierSignals) -> RouteTier { - let mut scores = [0_i32; 4]; - match str_field(&signals.raw, "turn_type") { - Some("chitchat" | "clarification" | "summarize") => scores[0] += 2, - Some("exploration" | "explanation" | "edit") => scores[1] += 2, - Some("planning" | "debug") => scores[2] += 2, - _ => {} - } - match str_field(&signals.raw, "code_modification_scope") { - Some("none" | "single_line") => scores[0] += 1, - Some("function" | "file") => scores[1] += 1, - Some("multi_file" | "cross_module") => scores[2] += 2, - _ => {} - } - let tool_count = u64_field(&signals.raw, "tool_call_count_estimate"); - if tool_count >= 4 { - scores[2] += 1; - } else if tool_count >= 2 { - scores[1] += 1; - } - if bool_field(&signals.raw, "requires_codebase_context") { - scores[1] += 1; - } - argmax_tier(scores) -} - -fn openclaw_policy_tier(signals: &ClassifierSignals) -> RouteTier { - let mut scores = [0_i32; 4]; - match str_field(&signals.raw, "turn_type") { - Some("chitchat" | "lookup" | "memory_recall" | "clarification") => scores[0] += 2, - Some("planning" | "explanation") => scores[1] += 2, - Some("tool_orchestration" | "action") => scores[2] += 2, - _ => {} - } - let tool_count = u64_field(&signals.raw, "tool_call_count_estimate"); - if tool_count >= 4 { - scores[2] += 1; - } else if tool_count >= 1 { - scores[1] += 1; - } - match str_field(&signals.raw, "memory_dependency") { - Some("heavy") => scores[2] += 2, - Some("light") => scores[1] += 1, - _ => {} - } - if bool_field(&signals.raw, "external_action_required") - && str_field(&signals.raw, "precision_requirement") == Some("high") - { - scores[2] += 2; - } else if bool_field(&signals.raw, "external_action_required") { - scores[1] += 1; - } - match str_field(&signals.raw, "ambiguity") { - Some("high") => scores[2] += 1, - Some("medium") => scores[1] += 1, - _ => {} - } - if str_field(&signals.raw, "channel_kind") == Some("casual") { - scores[0] += 1; - } else { - scores[2] += 1; - } - argmax_tier(scores) -} - -fn argmax_tier(scores: [i32; 4]) -> RouteTier { - let best = scores.into_iter().max().unwrap_or(0); - for (index, score) in scores.into_iter().enumerate().rev() { - if score == best { - return match index { - 0 => RouteTier::Simple, - 1 => RouteTier::Medium, - 2 => RouteTier::Complex, - _ => RouteTier::Reasoning, - }; - } - } - RouteTier::Medium -} - -fn classifier_tool_arguments(response: &ChatResponse, tool_name: &str) -> Result { - let body = response.body().ok_or_else(|| { - SwitchyardError::Backend("classifier returned a streaming response".to_string()) - })?; - let tool_calls = body - .get("choices") - .and_then(Value::as_array) - .and_then(|choices| choices.first()) - .and_then(|choice| choice.get("message")) - .and_then(|message| message.get("tool_calls")) - .and_then(Value::as_array) - .ok_or_else(|| { - SwitchyardError::Backend("classifier response did not include tool_calls".to_string()) - })?; - let Some(tool_call) = tool_calls.iter().find(|tool_call| { - tool_call - .get("function") - .and_then(|function| function.get("name")) - .and_then(Value::as_str) - == Some(tool_name) - }) else { - return Err(SwitchyardError::Backend(format!( - "classifier response did not call required tool {tool_name:?}" - ))); - }; - let arguments = tool_call - .get("function") - .and_then(|function| function.get("arguments")) - .ok_or_else(|| { - SwitchyardError::Backend("classifier tool call omitted arguments".to_string()) - })?; - match arguments { - Value::String(raw) if !raw.trim().is_empty() => Ok(raw.trim().to_string()), - Value::Object(_) => serde_json::to_string(arguments).map_err(|error| { - SwitchyardError::Backend(format!( - "classifier tool arguments could not be serialized: {error}" - )) - }), - _ => Err(SwitchyardError::Backend( - "classifier tool arguments must be a non-empty JSON string or object".to_string(), - )), - } -} - -fn summarize_request(request: &ChatRequest, recent_turn_window: usize) -> String { - let body = request.body(); - let summary_body = body - .as_object() - .map(|object| condense_body(object, recent_turn_window)) - .unwrap_or_else(|| body.clone()); - let payload = json!({ - "request_type": request.request_type(), - "body": summary_body, - }); - let text = serde_json::to_string(&payload).unwrap_or_else(|_| "{}".to_string()); - const MAX_CHARS: usize = 16_000; - if text.len() <= MAX_CHARS { - text - } else { - format!( - "{}...", - truncate_at_char_boundary(&text, MAX_CHARS - 32) - ) - } -} - -fn truncate_at_char_boundary(text: &str, max_bytes: usize) -> &str { - if text.len() <= max_bytes { - return text; - } - let mut end = max_bytes.min(text.len()); - while !text.is_char_boundary(end) { - end -= 1; - } - &text[..end] -} - -fn condense_body(body: &Map, recent_turn_window: usize) -> Value { - let mut out = Map::new(); - for (key, value) in body { - if !matches!(key.as_str(), "messages" | "tools" | "input" | "tool_choice") { - out.insert(key.clone(), value.clone()); - } - } - if let Some(Value::Array(tools)) = body.get("tools") { - out.insert( - "tools".to_string(), - Value::Array(tools.iter().map(condense_tool).collect()), - ); - } - if let Some(Value::Array(messages)) = body.get("messages") { - out.insert( - "messages".to_string(), - Value::Array(trim_messages(messages, recent_turn_window)), - ); - } - if let Some(input) = body.get("input") { - match input { - Value::Array(items) => { - out.insert( - "input".to_string(), - Value::Array(trim_messages(items, recent_turn_window)), - ); - } - Value::String(_) => { - out.insert("input".to_string(), input.clone()); - } - _ => {} - } - } - Value::Object(out) -} - -fn condense_tool(tool: &Value) -> Value { - let Some(object) = tool.as_object() else { - return tool.clone(); - }; - let mut out = object.clone(); - if let Some(Value::Object(function)) = object.get("function") { - let mut slim = function.clone(); - slim.remove("parameters"); - out.insert("function".to_string(), Value::Object(slim)); - } - out.remove("input_schema"); - Value::Object(out) -} - -fn trim_messages(messages: &[Value], recent_turn_window: usize) -> Vec { - let mut system = Vec::new(); - let mut first_user = None; - let mut first_user_idx = None; - for (idx, message) in messages.iter().enumerate() { - let role = message - .as_object() - .and_then(|object| object.get("role")) - .and_then(Value::as_str); - match role { - Some("system" | "developer") => system.push(message.clone()), - Some("user") if first_user.is_none() => { - first_user = Some(message.clone()); - first_user_idx = Some(idx); - } - _ => {} - } - } - let Some(first_user) = first_user else { - return system; - }; - let tail = messages - .iter() - .enumerate() - .filter(|(idx, message)| { - *idx > first_user_idx.unwrap_or(0) - && message - .as_object() - .and_then(|object| object.get("role")) - .and_then(Value::as_str) - .is_some_and(|role| !matches!(role, "system" | "developer")) - }) - .map(|(_, message)| message.clone()) - .collect::>(); - if recent_turn_window == 0 { - let mut out = system; - out.push(first_user); - if let Some(last_user) = tail.iter().rev().find(|message| { - message - .as_object() - .and_then(|object| object.get("role")) - .and_then(Value::as_str) - == Some("user") - }) { - out.push(last_user.clone()); - } - return out; - } - let mut out = system; - out.push(first_user); - let start = tail.len().saturating_sub(recent_turn_window); - out.extend_from_slice(&tail[start..]); - out -} - -fn base_properties(extra: Vec<(&'static str, Value)>) -> Map { - let mut properties = Map::from_iter([ - ( - "recommended_tier".to_string(), - enum_schema(&["simple", "medium", "complex", "reasoning"]), - ), - ("confidence".to_string(), json!({"type": "number"})), - ("abstain".to_string(), json!({"type": "boolean"})), - ]); - for (name, value) in extra { - properties.insert(name.to_string(), value); - } - properties -} - -fn object_schema(properties: Map) -> Value { - let required = properties - .keys() - .cloned() - .map(Value::String) - .collect::>(); - json!({ - "type": "object", - "properties": properties, - "required": required, - "additionalProperties": false, - }) -} - -fn enum_schema(values: &[&str]) -> Value { - json!({ - "type": "string", - "enum": values, - }) -} - -fn general_schema() -> Value { - object_schema(base_properties(vec![ - ( - "task_type", - enum_schema(&[ - "chat", - "summarization", - "extraction", - "translation", - "coding", - "debugging", - "math", - "planning", - "creative_writing", - "agentic_task", - "research", - "data_analysis", - "other", - ]), - ), - ( - "complexity", - enum_schema(&["simple", "medium", "complex", "reasoning"]), - ), - ( - "reasoning_depth", - enum_schema(&["none", "light", "multi_step", "deep"]), - ), - ("tool_planning_required", json!({"type": "boolean"})), - ( - "precision_requirement", - enum_schema(&["low", "medium", "high"]), - ), - ( - "context_dependency", - enum_schema(&["latest_message", "conversation", "external_context"]), - ), - ( - "structured_output_risk", - enum_schema(&["low", "medium", "high"]), - ), - ( - "reason_code", - enum_schema(&[ - "simple_qa", - "summarization", - "extraction", - "translation", - "coding_simple", - "coding_complex", - "debugging", - "math_reasoning", - "tool_agentic", - "long_context", - "structured_output", - "creative_generation", - "research_synthesis", - "ambiguous", - "other", - ]), - ), - ])) -} - -fn coding_agent_schema() -> Value { - object_schema(base_properties(vec![ - ( - "turn_type", - enum_schema(&[ - "chitchat", - "planning", - "exploration", - "edit", - "debug", - "explanation", - "clarification", - "summarize", - ]), - ), - ( - "code_modification_scope", - enum_schema(&[ - "none", - "single_line", - "function", - "file", - "multi_file", - "cross_module", - ]), - ), - ("tool_call_count_estimate", json!({"type": "integer"})), - ("requires_codebase_context", json!({"type": "boolean"})), - ])) -} - -fn openclaw_schema() -> Value { - object_schema(base_properties(vec![ - ( - "turn_type", - enum_schema(&[ - "chitchat", - "lookup", - "memory_recall", - "planning", - "tool_orchestration", - "action", - "explanation", - "clarification", - ]), - ), - ("tool_call_count_estimate", json!({"type": "integer"})), - ( - "memory_dependency", - enum_schema(&["none", "light", "heavy"]), - ), - ("external_action_required", json!({"type": "boolean"})), - ( - "precision_requirement", - enum_schema(&["low", "medium", "high"]), - ), - ("ambiguity", enum_schema(&["low", "medium", "high"])), - ("channel_kind", enum_schema(&["casual", "deliberate"])), - ])) -} - -fn str_field<'a>(value: &'a Value, name: &str) -> Option<&'a str> { - value.get(name).and_then(Value::as_str) -} - -fn bool_field(value: &Value, name: &str) -> bool { - value.get(name).and_then(Value::as_bool).unwrap_or(false) -} - -fn u64_field(value: &Value, name: &str) -> u64 { - value.get(name).and_then(Value::as_u64).unwrap_or(0) -} - -fn classifier_error(error: SwitchyardError) -> SwitchyardError { - SwitchyardError::Backend(format!( - "LLM classifier failed to produce valid route signals: {error}" - )) -} - -fn validate_config(config: &LlmRoutingProfileConfig) -> Result<()> { - ClassifierPolicy::from_name(&config.profile_name)?; - if !(config.classifier_min_confidence.is_finite() - && (0.0..=1.0).contains(&config.classifier_min_confidence)) - { - return Err(SwitchyardError::InvalidConfig(format!( - "classifier_min_confidence must be finite and in [0.0, 1.0], got {:?}", - config.classifier_min_confidence - ))); - } - if !(config.alignment_min_confidence.is_finite() - && (0.0..=1.0).contains(&config.alignment_min_confidence)) - { - return Err(SwitchyardError::InvalidConfig(format!( - "alignment_min_confidence must be finite and in [0.0, 1.0], got {:?}", - config.alignment_min_confidence - ))); - } - if config.classifier_max_tokens == 0 { - return Err(SwitchyardError::InvalidConfig( - "classifier_max_tokens must be greater than 0".to_string(), - )); - } - if let Some(default_tier) = &config.default_tier { - validate_backend_tier("default_tier", default_tier)?; - } - if let Some(tier_mapping) = &config.tier_mapping { - validate_backend_tier("tier_mapping.simple", &tier_mapping.simple)?; - validate_backend_tier("tier_mapping.medium", &tier_mapping.medium)?; - validate_backend_tier("tier_mapping.complex", &tier_mapping.complex)?; - validate_backend_tier("tier_mapping.reasoning", &tier_mapping.reasoning)?; - } - if let Some(tool_name) = &config.classifier_tool_name { - validate_classifier_tool_name(tool_name)?; - } - if let Some(tool_parameters) = &config.classifier_tool_parameters { - validate_classifier_tool_parameters(tool_parameters)?; - } - if config.fallback_target_on_evict != config.strong.id - && config.fallback_target_on_evict != config.weak.id - { - return Err(SwitchyardError::InvalidConfig(format!( - "fallback_target_on_evict={} must match one of [{}, {}]", - config.fallback_target_on_evict, config.weak.id, config.strong.id - ))); - } - if config.classifier.format != BackendFormat::OpenAi { - return Err(SwitchyardError::InvalidConfig( - "llm-routing classifier target must use format: openai".to_string(), - )); - } - Ok(()) -} - -fn validate_backend_tier(field: &str, value: &str) -> Result<()> { - match value { - TIER_STRONG | TIER_WEAK => Ok(()), - other => Err(SwitchyardError::InvalidConfig(format!( - "{field} must be {TIER_STRONG:?} or {TIER_WEAK:?}, got {other:?}" - ))), - } -} - -fn validate_classifier_tool_name(value: &str) -> Result<()> { - let valid = !value.is_empty() - && value - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-')); - if valid { - Ok(()) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "classifier_tool_name must be non-empty and contain only ASCII letters, numbers, '_' or '-', got {value:?}" - ))) - } -} - -fn validate_classifier_tool_parameters(value: &Value) -> Result<()> { - validate_strict_object_schema("classifier_tool_parameters", value) -} - -fn validate_strict_object_schema(path: &str, value: &Value) -> Result<()> { - if value.get("type").and_then(Value::as_str) != Some("object") { - return Err(SwitchyardError::InvalidConfig(format!( - "{path} must be a strict JSON schema object with type: object" - ))); - } - if value.get("additionalProperties").and_then(Value::as_bool) != Some(false) { - return Err(SwitchyardError::InvalidConfig(format!( - "{path} must set additionalProperties: false when strict tool calls are enabled" - ))); - } - let properties = value - .get("properties") - .and_then(Value::as_object) - .ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "{path} must define object properties for strict tool calls" - )) - })?; - let required = value - .get("required") - .and_then(Value::as_array) - .ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "{path} must list all properties in required for strict tool calls" - )) - })?; - let required = required - .iter() - .map(|item| { - item.as_str().ok_or_else(|| { - SwitchyardError::InvalidConfig(format!( - "{path}.required must contain only property names" - )) - }) - }) - .collect::>>()?; - let property_names = properties - .keys() - .map(String::as_str) - .collect::>(); - if required != property_names { - return Err(SwitchyardError::InvalidConfig(format!( - "{path}.required must contain exactly every key from properties for strict tool calls" - ))); - } - for (name, schema) in properties { - validate_nested_strict_schema(&format!("{path}.properties.{name}"), schema)?; - } - Ok(()) -} - -fn validate_nested_strict_schema(path: &str, value: &Value) -> Result<()> { - if value.get("type").and_then(Value::as_str) == Some("object") - || value.get("properties").is_some() - { - validate_strict_object_schema(path, value)?; - } - if let Some(items) = value.get("items") { - validate_nested_strict_schema(&format!("{path}.items"), items)?; - } - for combinator in ["anyOf", "oneOf", "allOf"] { - if let Some(schemas) = value.get(combinator).and_then(Value::as_array) { - for (index, schema) in schemas.iter().enumerate() { - validate_nested_strict_schema(&format!("{path}.{combinator}[{index}]"), schema)?; - } - } - } - Ok(()) -} - -fn resolve_default_tier(value: Option<&str>) -> Result { - let value = value.unwrap_or(TIER_STRONG); - validate_backend_tier("default_tier", value)?; - Ok(value.to_string()) -} - -fn default_profile_name() -> String { - PROFILE_CODING_AGENT.to_string() -} - -fn default_classifier_fail_open() -> bool { - true -} - -fn default_recent_turn_window() -> usize { - 4 -} - -fn default_classifier_max_tokens() -> u64 { - DEFAULT_CLASSIFIER_MAX_TOKENS -} - -fn default_alignment_min_confidence() -> f64 { - DEFAULT_ALIGNMENT_MIN_CONFIDENCE -} - -fn default_classifier_tool_name() -> String { - DEFAULT_CLASSIFIER_TOOL_NAME.to_string() -} - -fn default_fallback_target_on_evict() -> LlmTargetId { - LlmTargetId::from_static(TIER_STRONG) -} - -fn llm_routing_target(mut target: LlmTarget) -> LlmTarget { - apply_deepseek_defaults(&mut target); - target -} - -fn llm_routing_classifier_target(mut target: LlmTarget) -> LlmTarget { - if target.extra_body.is_none() && model_accepts_reasoning_hint(target.model.as_str()) { - target.extra_body = Some(json!({"chat_template_kwargs": {"enable_thinking": false}})); - } - apply_deepseek_defaults(&mut target); - target -} - -fn apply_deepseek_defaults(target: &mut LlmTarget) { - let model = target.model.as_str().to_ascii_lowercase(); - if target.extra_body.is_none() && model.contains("deepseek-v4") { - target.extra_body = Some(json!({"chat_template_kwargs": {"enable_thinking": false}})); - } - if target.extra_headers.is_empty() && model.contains("deepseek") { - target - .extra_headers - .insert("X-Inference-Priority".to_string(), "batch".to_string()); - } -} - -fn model_accepts_reasoning_hint(model: &str) -> bool { - let lowered = model.to_ascii_lowercase(); - !["anthropic", "bedrock", "claude"] - .iter() - .any(|tag| lowered.contains(tag)) -} - -fn normalize_reasoning_effort(request: &mut ChatRequest) { - const VALID: &[&str] = &["low", "medium", "high", "max"]; - let Value::Object(body) = request.body_mut() else { - return; - }; - let Some(effort) = body.get("reasoning_effort").and_then(Value::as_str) else { - return; - }; - if VALID.contains(&effort) { - return; - } - body.insert( - "reasoning_effort".to_string(), - Value::String("high".to_string()), - ); -} diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md deleted file mode 100644 index 80855374..00000000 --- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md +++ /dev/null @@ -1 +0,0 @@ -You are a routing classifier inside a coding-agent harness. Rate the turn, then call the provided route-selection tool exactly once with the requested schema. diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md deleted file mode 100644 index b66d7789..00000000 --- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md +++ /dev/null @@ -1 +0,0 @@ -You are a routing classifier for an LLM proxy. Call the provided route-selection tool exactly once with the requested schema. diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md deleted file mode 100644 index 2abb8605..00000000 --- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md +++ /dev/null @@ -1 +0,0 @@ -You are a routing classifier inside an OpenClaw personal-assistant agent. Rate the message, then call the provided route-selection tool exactly once with the requested schema. diff --git a/crates/switchyard-components-v2/src/profiles/macros.rs b/crates/switchyard-components-v2/src/profiles/macros.rs deleted file mode 100644 index 6f6bcfcb..00000000 --- a/crates/switchyard-components-v2/src/profiles/macros.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Macros used by profile implementations and profile config wiring. - -macro_rules! profile_types { - ($($config:ident),+ $(,)?) => { - /// Resolved, strongly typed config for one profile entry. - /// - /// Variants intentionally mirror config type names so this macro has one - /// input per profile and does not need a second variant-name registry. - #[allow(clippy::enum_variant_names)] - #[derive(Clone, Debug, PartialEq)] - pub(crate) enum ProfileConfigEntry { - $( - /// Config variant generated from the profile registry. - $config(Box<$config>), - )+ - } - - impl ProfileConfigEntry { - /// Returns the file-facing type discriminator for this resolved profile config. - pub(crate) fn profile_type(&self) -> &'static str { - match self { - $( - Self::$config(_) => - <$config as crate::config::ProfileConfigDefinition>::PROFILE_TYPE, - )+ - } - } - - /// Builds this resolved config into the erased runtime profile. - pub(crate) fn build_boxed( - &self, - ) -> switchyard_core::Result> { - match self { - $( - Self::$config(config) => - <$config as crate::ProfileConfig>::build_boxed(config.as_ref()), - )+ - } - } - } - - /// Parses a serialized profile body by dispatching to the owning config type. - /// - /// The `type` discriminator is matched with `-` and `_` treated as - /// equivalent, so a snake_case name copied from a legacy route bundle - /// (e.g. `random_routing`) resolves to its v2 profile, and a hyphenated - /// spelling of an underscore type (e.g. `stage-router`) resolves too. - pub(crate) fn parse_profile_config( - profile_type: &str, - value: serde_json::Value, - env: &crate::config::ProfileBuildEnv<'_>, - ) -> switchyard_core::Result { - // Compare treating `-` and `_` as equal without allocating. Profile - // type names are short ASCII, so an equal-length per-byte scan that - // folds the separators is enough. Registered names must stay unique - // under this folding (they are today) so dispatch is unambiguous. - fn eq_ignoring_separators(a: &str, b: &str) -> bool { - a.len() == b.len() - && a.bytes().zip(b.bytes()).all(|(x, y)| { - x == y || (matches!(x, b'-' | b'_') && matches!(y, b'-' | b'_')) - }) - } - - $( - if eq_ignoring_separators( - profile_type, - <$config as crate::config::ProfileConfigDefinition>::PROFILE_TYPE, - ) { - let config = - <$config as crate::config::ProfileConfigDefinition>::parse_profile_config( - value, - env, - )?; - return Ok(ProfileConfigEntry::$config(Box::new(config))); - } - )+ - Err(switchyard_core::SwitchyardError::InvalidConfig(format!( - "unknown profile type `{profile_type}`" - ))) - } - }; -} - -pub(crate) use profile_types; diff --git a/crates/switchyard-components-v2/src/profiles/mod.rs b/crates/switchyard-components-v2/src/profiles/mod.rs deleted file mode 100644 index f8396002..00000000 --- a/crates/switchyard-components-v2/src/profiles/mod.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Profile implementations in the flatter components-v2 design. - -mod latency_service; -mod llm_routing; -mod macros; -mod noop; -mod passthrough; -mod profile_types; -mod random_routing; -mod stage_router; -mod subagent_override; - -pub use latency_service::{ - EndpointHealth, EndpointHealthStatus, LatencyServiceProcessedRequest, LatencyServiceProfile, - LatencyServiceProfileConfig, SelectedTarget, -}; -pub use llm_routing::{ - LlmRoutingDecision, LlmRoutingProcessedRequest, LlmRoutingProfile, LlmRoutingProfileConfig, - LlmRoutingTierMapping, -}; -pub use noop::{NoopProfile, NoopProfileConfig}; -pub use passthrough::{PassthroughProfile, PassthroughProfileConfig}; -pub(crate) use profile_types::{parse_profile_config, ProfileConfigEntry}; -pub use random_routing::{ - RandomRoutingProcessedRequest, RandomRoutingProfile, RandomRoutingProfileConfig, -}; -pub use stage_router::{ - StageRouterClassifierConfig, StageRouterDecision, StageRouterDecisionSource, - StageRouterPickerMode, StageRouterProcessedRequest, StageRouterProfile, - StageRouterProfileConfig, StageRouterTier, -}; -pub(crate) use subagent_override::SubagentOverrideProfile; diff --git a/crates/switchyard-components-v2/src/profiles/noop.rs b/crates/switchyard-components-v2/src/profiles/noop.rs deleted file mode 100644 index efa509ce..00000000 --- a/crates/switchyard-components-v2/src/profiles/noop.rs +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! No-op profile for local smoke tests and benchmark harnesses. - -use async_trait::async_trait; -use serde_json::json; -use switchyard_core::{ChatResponse, Result}; - -use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse}; - -/// Config for a no-op profile. -#[profile_config("noop")] -pub struct NoopProfileConfig {} - -impl ProfileConfig for NoopProfileConfig { - type Runtime = NoopProfile; - - /// Builds the no-op runtime profile. - fn build(&self) -> Result { - Ok(NoopProfile {}) - } -} - -/// Profile that returns a deterministic local response without calling an upstream model. -pub struct NoopProfile {} - -#[async_trait] -impl ProfileHooks for NoopProfile { - type ProcessedRequest = ProfileInput; - - /// Leaves the request unchanged for hook-level inspection. - async fn process(&self, input: ProfileInput) -> Result { - Ok(input) - } - - /// Leaves the response unchanged after no-op response creation. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for NoopProfile { - /// Returns a deterministic OpenAI-compatible chat completion. - async fn run(&self, input: ProfileInput) -> Result { - let processed = self.process(input).await?; - let model = processed.request.model().unwrap_or("switchyard/noop"); - let response = self - .rprocess( - &processed, - ChatResponse::openai_completion(json!({ - "id": "switchyard-noop", - "object": "chat.completion", - "model": model, - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": "ok" - }, - "finish_reason": "stop" - }], - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0 - } - })), - ) - .await?; - Ok(ProfileResponse::from(response)) - } -} diff --git a/crates/switchyard-components-v2/src/profiles/passthrough.rs b/crates/switchyard-components-v2/src/profiles/passthrough.rs deleted file mode 100644 index 8d9edb89..00000000 --- a/crates/switchyard-components-v2/src/profiles/passthrough.rs +++ /dev/null @@ -1,273 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Passthrough profile implemented as a single profile-owned runtime. - -use std::time::Instant; - -use async_trait::async_trait; -use switchyard_components::StatsAccumulator; -use switchyard_core::{ChatResponse, LlmTarget, Result}; - -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::stats_recording::record_usage_or_wrap_stream; -use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse}; - -/// Config for the flatter passthrough profile. -#[profile_config("passthrough")] -pub struct PassthroughProfileConfig { - /// Target served by this profile without routing. - #[profile_target] - pub target: LlmTarget, -} - -impl ProfileConfig for PassthroughProfileConfig { - type Runtime = PassthroughProfile; - - /// Builds the runtime profile using the existing native backend stack. - fn build(&self) -> Result { - Ok(PassthroughProfile { - backend: native_target_backend(self.target.clone())?, - stats: profile_stats_accumulator(), - }) - } -} - -/// Passthrough profile in the flatter design. -pub struct PassthroughProfile { - backend: TargetBackend, - stats: StatsAccumulator, -} - -#[async_trait] -impl ProfileHooks for PassthroughProfile { - type ProcessedRequest = ProfileInput; - - /// Rewrites the request model to the single configured target. - async fn process(&self, mut input: ProfileInput) -> Result { - input - .request - .set_model(self.backend.target().model.as_str()); - Ok(input) - } - - /// Leaves the response unchanged after the backend call. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for PassthroughProfile { - /// Executes passthrough by composing the hook methods around one backend call. - /// - /// Passthrough has no per-call routing decision to preserve, so `run()` can use the - /// straightforward `process -> backend -> rprocess` shape described by the - /// [`Profile`] trait. Profiles that select targets dynamically may use a different - /// internal flow while still exposing the same public lifecycle methods. - async fn run(&self, input: ProfileInput) -> Result { - let profile_started_at = Instant::now(); - let processed = self.process(input).await?; - let target_model = self.backend.target().model.clone(); - let backend_started_at = Instant::now(); - let response = match self.backend.call(&processed.request).await { - Ok(response) => response, - Err(error) => { - self.stats.record_error(target_model.as_str(), None)?; - return Err(error); - } - }; - let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; - self.stats - .record_success(target_model.to_string(), Some(backend_latency_ms), None)?; - let response = record_usage_or_wrap_stream( - &self.stats, - target_model.as_str(), - None, - profile_started_at, - backend_latency_ms, - response, - )?; - let response = self.rprocess(&processed, response).await?; - Ok(ProfileResponse::from(response)) - } -} - -#[cfg(test)] -mod tests { - use std::sync::{Arc, Mutex}; - - use async_trait::async_trait; - use serde_json::{json, Value}; - use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, SwitchyardError}; - - use crate::backend::{ProfileBackend, TargetBackend}; - use crate::RequestMetadata; - - use super::*; - - #[derive(Clone, Debug, PartialEq)] - struct ObservedCall { - body: Value, - } - - struct TestBackend { - calls: Arc>>, - } - - #[async_trait] - impl ProfileBackend for TestBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.calls - .lock() - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))? - .push(ObservedCall { - body: request.body().clone(), - }); - Ok(ChatResponse::openai_completion(json!({ - "model": request.model(), - "served_model": request.model(), - "usage": { - "prompt_tokens": 5, - "completion_tokens": 3, - }, - }))) - } - } - - fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) - } - - fn backend(target: &LlmTarget, calls: Arc>>) -> TargetBackend { - TargetBackend::new(target.clone(), Arc::new(TestBackend { calls })) - } - - fn observed(calls: &Arc>>) -> Result> { - calls - .lock() - .map(|calls| calls.clone()) - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string())) - } - - fn profile_input(request: ChatRequest) -> ProfileInput { - ProfileInput { - request, - metadata: RequestMetadata::default(), - } - } - - fn profile(target: LlmTarget) -> Result<(PassthroughProfile, Arc>>)> { - let calls = Arc::new(Mutex::new(Vec::new())); - let profile = PassthroughProfile { - backend: backend(&target, calls.clone()), - stats: StatsAccumulator::new(), - }; - Ok((profile, calls)) - } - - #[tokio::test] - async fn passthrough_profile_calls_single_target_backend() -> Result<()> { - let (profile, calls) = profile(target("direct", "provider/model")?)?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [{"role": "user", "content": "hi"}], - })))) - .await?; - - let response = response.response; - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].body["model"], "provider/model"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["served_model"], "provider/model"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - Ok(()) - } - - #[tokio::test] - async fn run_records_stats_for_single_target() -> Result<()> { - let (profile, _calls) = profile(target("direct", "provider/model")?)?; - - let _response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })))) - .await?; - - let snapshot = profile.stats.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!(snapshot.total_tokens.prompt, 5); - assert_eq!(snapshot.total_tokens.completion, 3); - let model = snapshot.models.get("provider/model").ok_or_else(|| { - SwitchyardError::Other("provider model stats should be present".into()) - })?; - assert_eq!(model.calls, 1); - assert_eq!(model.tier, None); - Ok(()) - } - - #[tokio::test] - async fn process_only_prepares_target_request_and_does_not_call_backend() -> Result<()> { - let (profile, calls) = profile(target("direct", "provider/model")?)?; - let request = ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })); - - let processed = profile.process(profile_input(request.clone())).await?; - - assert_eq!(processed.request.model(), Some("provider/model")); - assert_eq!( - processed.request.body()["messages"], - request.body()["messages"] - ); - assert!(observed(&calls)?.is_empty()); - Ok(()) - } - - #[tokio::test] - async fn rprocess_only_returns_response_unchanged() -> Result<()> { - let (profile, calls) = profile(target("direct", "provider/model")?)?; - let response = ChatResponse::openai_completion(json!({"ok": true})); - - let request = profile_input(ChatRequest::openai_chat( - json!({"model": "client/model", "messages": []}), - )); - let processed = profile.rprocess(&request, response).await?; - - match processed { - ChatResponse::OpenAiCompletion(body) => assert_eq!(body.body()["ok"], true), - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - assert!(observed(&calls)?.is_empty()); - Ok(()) - } - - #[tokio::test] - async fn malformed_body_still_gets_target_model_for_single_target_backend() -> Result<()> { - let (profile, calls) = profile(target("direct", "provider/model")?)?; - - let _response = profile - .run(profile_input(ChatRequest::openai_chat(json!("bad-body")))) - .await?; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].body, json!({"model": "provider/model"})); - Ok(()) - } -} diff --git a/crates/switchyard-components-v2/src/profiles/profile_types.rs b/crates/switchyard-components-v2/src/profiles/profile_types.rs deleted file mode 100644 index 3126a815..00000000 --- a/crates/switchyard-components-v2/src/profiles/profile_types.rs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Central profile config registry generated from concrete profile config types. - -use super::macros::profile_types; -use super::{ - LatencyServiceProfileConfig, LlmRoutingProfileConfig, NoopProfileConfig, - PassthroughProfileConfig, RandomRoutingProfileConfig, StageRouterProfileConfig, -}; - -profile_types! { - StageRouterProfileConfig, - PassthroughProfileConfig, - RandomRoutingProfileConfig, - LatencyServiceProfileConfig, - LlmRoutingProfileConfig, - NoopProfileConfig, -} diff --git a/crates/switchyard-components-v2/src/profiles/random_routing.rs b/crates/switchyard-components-v2/src/profiles/random_routing.rs deleted file mode 100644 index dea63c67..00000000 --- a/crates/switchyard-components-v2/src/profiles/random_routing.rs +++ /dev/null @@ -1,559 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Random-routing profile implemented as a single profile-owned runtime. - -use std::time::Instant; - -use async_trait::async_trait; -use switchyard_components::request_processors::{ - RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier, -}; -use switchyard_components::StatsAccumulator; -use switchyard_core::{ChatResponse, LlmTarget, Result, SwitchyardError}; - -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::stats_recording::record_usage_or_wrap_stream; -use crate::{ - profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse, - RoutingMetadata, -}; - -/// Config for the flatter random-routing profile. -#[profile_config("random-routing")] -pub struct RandomRoutingProfileConfig { - /// Strong target served by this profile. - #[profile_target] - pub strong: LlmTarget, - /// Weak target served by this profile. - #[profile_target] - pub weak: LlmTarget, - /// Probability of selecting the strong target. - #[serde(default = "default_strong_probability")] - pub strong_probability: f64, - /// Optional deterministic RNG seed for reproducible routing. - #[serde(default)] - pub rng_seed: Option, -} - -impl ProfileConfig for RandomRoutingProfileConfig { - type Runtime = RandomRoutingProfile; - - /// Builds the runtime profile using existing native backend construction. - fn build(&self) -> Result { - let router_config = - RandomRoutingProcessorConfig::new(self.strong.clone(), self.weak.clone()) - .with_strong_probability(self.strong_probability)? - .with_rng_seed(self.rng_seed); - Ok(RandomRoutingProfile { - router: RandomRoutingEngine::new(router_config)?, - strong_backend: native_target_backend(self.strong.clone())?, - weak_backend: native_target_backend(self.weak.clone())?, - stats: profile_stats_accumulator(), - }) - } -} - -/// Random-routing profile in the flatter design. -pub struct RandomRoutingProfile { - router: RandomRoutingEngine, - strong_backend: TargetBackend, - weak_backend: TargetBackend, - stats: StatsAccumulator, -} - -/// Processed random-routing request with the profile-owned routing decision. -pub struct RandomRoutingProcessedRequest { - /// Routed input prepared for the selected backend. - pub profile_input: ProfileInput, - /// Selected routing decision for this request. - pub decision: RandomRoutingDecision, -} - -impl RandomRoutingProfile { - // Selects the target and rewrites the request model without side-channel state. - fn route_request(&self, mut input: ProfileInput) -> Result { - let decision = self - .router - .select(input.request.model().map(std::borrow::ToOwned::to_owned))?; - input.request.set_model(decision.selected_model.as_str()); - Ok(RandomRoutingProcessedRequest { - profile_input: input, - decision, - }) - } - - // Finds the routed backend by the target ID emitted by the routing engine. - fn selected_backend(&self, decision: &RandomRoutingDecision) -> Result<&TargetBackend> { - if decision.selected_target == self.strong_backend.target().id { - Ok(&self.strong_backend) - } else if decision.selected_target == self.weak_backend.target().id { - Ok(&self.weak_backend) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "router selected target {} that is not configured for this profile", - decision.selected_target - ))) - } - } - - fn routing_metadata(&self, decision: &RandomRoutingDecision) -> RoutingMetadata { - let comparison = if decision.tier == RandomRoutingTier::Strong { - "<" - } else { - ">=" - }; - RoutingMetadata { - selected_model: Some(decision.selected_model.to_string()), - selected_tier: Some(decision.tier.as_str().to_string()), - confidence: None, - router_version: Some("random-routing:v1".to_string()), - tolerance: Some(decision.strong_probability), - rationale: Some(format!( - "random draw {} {comparison} strong_probability {}; selected {}", - decision.draw, - decision.strong_probability, - decision.tier.as_str() - )), - } - } -} - -#[async_trait] -impl ProfileHooks for RandomRoutingProfile { - type ProcessedRequest = RandomRoutingProcessedRequest; - - /// Performs a standalone routing rewrite for hook-level inspection. - async fn process(&self, input: ProfileInput) -> Result { - self.route_request(input) - } - - /// Leaves the backend response unchanged after random routing completes. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for RandomRoutingProfile { - /// Executes random routing while keeping selected-target state local to this call. - async fn run(&self, input: ProfileInput) -> Result { - let profile_started_at = Instant::now(); - let processed = self.process(input).await?; - let decision = &processed.decision; - let selected_backend = self.selected_backend(decision)?; - let backend_started_at = Instant::now(); - let response = match selected_backend - .call(&processed.profile_input.request) - .await - { - Ok(response) => response, - Err(error) => { - self.stats.record_error( - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - )?; - return Err(error); - } - }; - let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; - self.stats.record_success( - decision.selected_model.as_str(), - Some(backend_latency_ms), - Some(decision.tier.as_str()), - )?; - let response = record_usage_or_wrap_stream( - &self.stats, - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - profile_started_at, - backend_latency_ms, - response, - )?; - let response = self.rprocess(&processed, response).await?; - Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(decision), - )) - } -} - -/// Default probability matches the existing random-routing config. -fn default_strong_probability() -> f64 { - 0.5 -} - -#[cfg(test)] -mod tests { - use std::sync::{Arc, Mutex}; - - use async_trait::async_trait; - use futures_util::StreamExt; - use serde_json::{json, Value}; - use switchyard_core::{ - BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent, SwitchyardError, - }; - - use crate::backend::{ProfileBackend, TargetBackend}; - use crate::RequestMetadata; - - use super::*; - - #[derive(Clone, Debug, PartialEq)] - struct ObservedCall { - backend: &'static str, - body: Value, - } - - struct TestBackend { - name: &'static str, - calls: Arc>>, - } - - struct StreamingUsageBackend; - - #[async_trait] - impl ProfileBackend for TestBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.calls - .lock() - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))? - .push(ObservedCall { - backend: self.name, - body: request.body().clone(), - }); - Ok(ChatResponse::openai_completion(json!({ - "served_by": self.name, - "model": request.model(), - "usage": { - "prompt_tokens": 11, - "completion_tokens": 7, - }, - }))) - } - } - - #[async_trait] - impl ProfileBackend for StreamingUsageBackend { - async fn call(&self, _request: &ChatRequest) -> Result { - Ok(ChatResponse::OpenAiStream(Box::pin( - futures_util::stream::iter([ - Ok(StreamEvent::Json(json!({ - "choices": [{"delta": {"content": "ok"}}], - }))), - Ok(StreamEvent::Json(json!({ - "choices": [], - "usage": { - "prompt_tokens": 3, - "completion_tokens": 4, - "total_tokens": 7, - }, - }))), - ]), - ))) - } - } - - fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) - } - - fn config(strong: LlmTarget, weak: LlmTarget, probability: f64) -> RandomRoutingProfileConfig { - RandomRoutingProfileConfig { - strong, - weak, - strong_probability: probability, - rng_seed: Some(7), - } - } - - fn backend( - strong: &LlmTarget, - weak: &LlmTarget, - calls: Arc>>, - ) -> (TargetBackend, TargetBackend) { - ( - TargetBackend::new( - strong.clone(), - Arc::new(TestBackend { - name: "strong-backend", - calls: calls.clone(), - }), - ), - TargetBackend::new( - weak.clone(), - Arc::new(TestBackend { - name: "weak-backend", - calls, - }), - ), - ) - } - - fn observed(calls: &Arc>>) -> Result> { - calls - .lock() - .map(|calls| calls.clone()) - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string())) - } - - fn profile_input(request: ChatRequest) -> ProfileInput { - ProfileInput { - request, - metadata: RequestMetadata::default(), - } - } - - fn profile( - strong: LlmTarget, - weak: LlmTarget, - probability: f64, - ) -> Result<(RandomRoutingProfile, Arc>>)> { - let calls = Arc::new(Mutex::new(Vec::new())); - let config = config(strong.clone(), weak.clone(), probability); - let router_config = - RandomRoutingProcessorConfig::new(config.strong.clone(), config.weak.clone()) - .with_strong_probability(config.strong_probability)? - .with_rng_seed(config.rng_seed); - let (strong_backend, weak_backend) = backend(&strong, &weak, calls.clone()); - let profile = RandomRoutingProfile { - router: RandomRoutingEngine::new(router_config)?, - strong_backend, - weak_backend, - stats: StatsAccumulator::new(), - }; - Ok((profile, calls)) - } - - #[tokio::test] - async fn random_routing_profile_routes_with_request_only_handoff() -> Result<()> { - let (profile, calls) = profile( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.0, - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [{"role": "user", "content": "hi"}], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong")); - assert_eq!( - routing_metadata.router_version.as_deref(), - Some("random-routing:v1") - ); - assert_eq!(routing_metadata.tolerance, Some(1.0)); - let response = response.response; - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "strong-backend"); - assert_eq!(calls[0].body["model"], "frontier/model"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["served_by"], "strong-backend"); - assert_eq!(body.body()["model"], "frontier/model"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - Ok(()) - } - - #[tokio::test] - async fn run_records_stats_with_selected_random_tier() -> Result<()> { - let (profile, _calls) = profile( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.0, - )?; - - let _response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })))) - .await?; - - let snapshot = profile.stats.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!(snapshot.total_tokens.prompt, 11); - assert_eq!(snapshot.total_tokens.completion, 7); - let model = snapshot.models.get("frontier/model").ok_or_else(|| { - SwitchyardError::Other("frontier model stats should be present".into()) - })?; - assert_eq!(model.calls, 1); - assert_eq!(model.tier.as_deref(), Some("strong")); - let tier = snapshot - .tiers - .get("strong") - .ok_or_else(|| SwitchyardError::Other("strong tier stats should be present".into()))?; - assert_eq!(tier.calls, 1); - assert_eq!(tier.model, "frontier/model"); - Ok(()) - } - - #[tokio::test] - async fn run_records_streaming_usage_with_selected_random_tier() -> Result<()> { - let strong = target("strong", "frontier/model")?; - let weak = target("weak", "cheap/model")?; - let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone()) - .with_strong_probability(0.0)? - .with_rng_seed(Some(7)); - let profile = RandomRoutingProfile { - router: RandomRoutingEngine::new(router_config)?, - strong_backend: TargetBackend::new( - strong, - Arc::new(TestBackend { - name: "strong-backend", - calls: Arc::new(Mutex::new(Vec::new())), - }), - ), - weak_backend: TargetBackend::new(weak, Arc::new(StreamingUsageBackend)), - stats: StatsAccumulator::new(), - }; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - "stream": true, - })))) - .await?; - let ChatResponse::OpenAiStream(mut stream) = response.response else { - return Err(SwitchyardError::Other("expected streaming response".into())); - }; - while let Some(event) = stream.next().await { - event?; - } - - let snapshot = profile.stats.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!(snapshot.total_tokens.prompt, 3); - assert_eq!(snapshot.total_tokens.completion, 4); - assert_eq!(snapshot.total_tokens.total, 7); - let model = snapshot - .models - .get("cheap/model") - .ok_or_else(|| SwitchyardError::Other("weak model stats should be present".into()))?; - assert_eq!(model.calls, 1); - assert_eq!(model.tier.as_deref(), Some("weak")); - assert_eq!(model.total_tokens, 7); - let tier = snapshot - .tiers - .get("weak") - .ok_or_else(|| SwitchyardError::Other("weak tier stats should be present".into()))?; - assert_eq!(tier.total_tokens, 7); - Ok(()) - } - - #[tokio::test] - async fn run_disambiguates_duplicate_target_models_without_context_state() -> Result<()> { - let (profile, calls) = profile( - target("strong-endpoint", "shared/model")?, - target("weak-endpoint", "shared/model")?, - 0.0, - )?; - - let _response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })))) - .await?; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "weak-backend"); - assert_eq!(calls[0].body["model"], "shared/model"); - Ok(()) - } - - #[tokio::test] - async fn malformed_request_body_is_recovered_without_context_state() -> Result<()> { - let (profile, calls) = profile( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.0, - )?; - - let _response = profile - .run(profile_input(ChatRequest::openai_chat(json!("bad-body")))) - .await?; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "strong-backend"); - assert_eq!(calls[0].body, json!({"model": "frontier/model"})); - Ok(()) - } - - #[tokio::test] - async fn process_only_prepares_request_and_does_not_call_backend() -> Result<()> { - let (profile, calls) = profile( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.0, - )?; - - let request = profile - .process(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })))) - .await?; - - assert_eq!( - request.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!(request.decision.selected_model.as_str(), "frontier/model"); - assert!(observed(&calls)?.is_empty()); - Ok(()) - } - - #[tokio::test] - async fn rprocess_only_handles_response() -> Result<()> { - let (profile, calls) = profile( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.0, - )?; - - let processed = profile - .process(profile_input(ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [], - })))) - .await?; - let response = profile - .rprocess( - &processed, - ChatResponse::openai_completion(json!({"ok": true})), - ) - .await?; - - match response { - ChatResponse::OpenAiCompletion(body) => assert_eq!(body.body()["ok"], true), - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - assert!(observed(&calls)?.is_empty()); - Ok(()) - } -} diff --git a/crates/switchyard-components-v2/src/profiles/stage_router.rs b/crates/switchyard-components-v2/src/profiles/stage_router.rs deleted file mode 100644 index 6aebe709..00000000 --- a/crates/switchyard-components-v2/src/profiles/stage_router.rs +++ /dev/null @@ -1,1362 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Signal stage_router profile implemented as a profile-owned Rust runtime. - -use std::time::{Duration, Instant}; - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use switchyard_components::dimension_collector::{ - extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, -}; -use switchyard_components::stats::{usage_from_body, TokenUsage}; -use switchyard_components::StatsAccumulator; -use switchyard_core::{ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError}; - -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::stats_recording::record_usage_or_wrap_stream; -use crate::{ - profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse, - RoutingMetadata, -}; - -const DEFAULT_CONFIDENCE_THRESHOLD: f64 = 0.7; -const DEFAULT_CLASSIFIER_TIMEOUT_SECS: f64 = 30.0; -const DEFAULT_CLASSIFIER_RECENT_TURN_WINDOW: usize = 3; -const CLASSIFIER_MAX_TOKENS: u32 = 4096; -const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; -const STAGE_ROUTER_PROFILE_TYPE: &str = "stage_router"; - -const SEVERITY_CRITICAL: f32 = 1.0; -const CLEAN_TESTS_MIN_TURN_DEPTH: u32 = 10; -const CLEAN_TESTS_MAX_WRITES: u32 = 1; -const PURE_BASH_NORM: f64 = 8.0; - -const CLASSIFIER_SYSTEM_PROMPT: &str = include_str!("stage_router/prompts/classifier.md"); - -const DEFAULT_WEIGHTS: &[(&str, f64)] = &[ - ("severity", 0.80), - ("stuck_exploring", 0.70), - ("no_progress", 0.60), - ("tests_passed", -0.80), - ("planning_active", -0.70), - ("write_intensity", -0.40), - ("edit_intensity", -0.30), - ("recent_write_intensity", -0.30), - ("pure_bash_intensity", -0.30), - ("no_error_streak_intensity", -0.20), -]; - -/// Default picker mode names the tier used when the scorer is ambiguous. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StageRouterPickerMode { - /// Default to capable unless the scorer/classifier confidently picks efficient. - CapableFirst, - /// Default to efficient unless the scorer/classifier confidently picks capable. - EfficientFirst, -} - -impl StageRouterPickerMode { - fn default_tier(self) -> StageRouterTier { - match self { - Self::CapableFirst => StageRouterTier::Capable, - Self::EfficientFirst => StageRouterTier::Efficient, - } - } -} - -/// Optional LLM classifier invoked for low-confidence scorer outputs. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StageRouterClassifierConfig { - /// OpenAI-compatible classifier model. - pub model: String, - /// API key used for the classifier call. - pub api_key: String, - /// OpenAI-compatible base URL. Defaults to OpenAI's `/v1` endpoint. - #[serde(default)] - pub base_url: Option, - /// Per-call timeout in seconds. - #[serde(default = "default_classifier_timeout_secs")] - pub timeout_secs: f64, - /// Number of trailing request messages rendered into the classifier prompt. - #[serde(default = "default_classifier_recent_turn_window")] - pub recent_turn_window: usize, - /// Maximum tokens allowed for the classifier response. - #[serde(default = "default_classifier_max_tokens")] - pub max_tokens: u32, - /// Optional system prompt override for the classifier request. - #[serde(default)] - pub system_prompt: Option, -} - -/// Config for a capable/efficient signal stage_router profile. -#[profile_config("stage_router")] -pub struct StageRouterProfileConfig { - /// Capable target served by this profile. - #[profile_target] - pub capable: LlmTarget, - /// Efficient target served by this profile. - #[profile_target] - pub efficient: LlmTarget, - /// Target used for one retry after context-window overflow. - pub fallback_target_on_evict: LlmTargetId, - /// Picker mode controlling the low-confidence default tier. - #[serde(default = "default_picker")] - pub picker: StageRouterPickerMode, - /// Scorer confidence threshold in `[0.0, 1.0]`. - #[serde(default = "default_confidence_threshold")] - pub confidence_threshold: f64, - /// Sliding window for `recent_*` tool-result signal counts. - #[serde(default = "default_signal_recent_window")] - pub signal_recent_window: usize, - /// Optional LLM classifier for ambiguous scorer outputs. - #[serde(default)] - pub classifier: Option, - /// Whether to emit stats for this profile. - #[serde(default = "default_enable_stats")] - pub enable_stats: bool, -} - -impl ProfileConfig for StageRouterProfileConfig { - type Runtime = StageRouterProfile; - - /// Builds the runtime profile using native target backends. - fn build(&self) -> Result { - self.validate()?; - Ok(StageRouterProfile { - capable_backend: native_target_backend(self.capable.clone())?, - efficient_backend: native_target_backend(self.efficient.clone())?, - fallback_target_on_evict: self.fallback_target_on_evict.clone(), - picker: self.picker, - confidence_threshold: self.confidence_threshold, - signal_recent_window: self.signal_recent_window, - classifier: self - .classifier - .as_ref() - .map(StageRouterTierClassifier::new) - .transpose()?, - stats: profile_stats_accumulator(), - enable_stats: self.enable_stats, - }) - } -} - -impl StageRouterProfileConfig { - fn validate(&self) -> Result<()> { - if self.capable.id == self.efficient.id { - return Err(SwitchyardError::InvalidConfig( - "stage_router capable and efficient targets must have distinct target ids" - .to_string(), - )); - } - if !self.confidence_threshold.is_finite() - || !(0.0..=1.0).contains(&self.confidence_threshold) - { - return Err(SwitchyardError::InvalidConfig(format!( - "confidence_threshold must be finite and in [0.0, 1.0], got {:?}", - self.confidence_threshold - ))); - } - if self.signal_recent_window == 0 { - return Err(SwitchyardError::InvalidConfig( - "signal_recent_window must be at least 1".to_string(), - )); - } - if self.fallback_target_on_evict != self.capable.id - && self.fallback_target_on_evict != self.efficient.id - { - return Err(SwitchyardError::InvalidConfig(format!( - "fallback_target_on_evict={} must match one of [{}, {}]", - self.fallback_target_on_evict, self.efficient.id, self.capable.id - ))); - } - if let Some(classifier) = &self.classifier { - classifier.validate()?; - } - Ok(()) - } -} - -impl StageRouterClassifierConfig { - fn validate(&self) -> Result<()> { - if self.model.trim().is_empty() { - return Err(SwitchyardError::InvalidConfig( - "classifier.model must not be empty".to_string(), - )); - } - if !self.timeout_secs.is_finite() || self.timeout_secs <= 0.0 { - return Err(SwitchyardError::InvalidConfig(format!( - "classifier.timeout_secs must be finite and > 0.0, got {:?}", - self.timeout_secs - ))); - } - if self.max_tokens == 0 { - return Err(SwitchyardError::InvalidConfig( - "classifier.max_tokens must be greater than 0".to_string(), - )); - } - if let Some(system_prompt) = &self.system_prompt { - if system_prompt.trim().is_empty() { - return Err(SwitchyardError::InvalidConfig( - "classifier.system_prompt must not be empty".to_string(), - )); - } - } - Ok(()) - } -} - -/// Capable/efficient stage_router profile runtime. -pub struct StageRouterProfile { - capable_backend: TargetBackend, - efficient_backend: TargetBackend, - fallback_target_on_evict: LlmTargetId, - picker: StageRouterPickerMode, - confidence_threshold: f64, - signal_recent_window: usize, - classifier: Option, - stats: StatsAccumulator, - enable_stats: bool, -} - -/// Processed stage_router request with profile-owned decision state. -pub struct StageRouterProcessedRequest { - /// Routed input prepared for the selected backend. - pub profile_input: ProfileInput, - /// Selected routing decision for this request. - pub decision: StageRouterDecision, -} - -/// Named side of a stage_router decision. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StageRouterTier { - /// Capable tier. - Capable, - /// Efficient tier. - Efficient, -} - -impl StageRouterTier { - /// Stable lowercase label used by stats. - pub fn as_str(self) -> &'static str { - match self { - Self::Capable => "capable", - Self::Efficient => "efficient", - } - } -} - -/// Source that produced a stage_router decision. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StageRouterDecisionSource { - /// Hard override fired. - Override, - /// Dimension scorer crossed `confidence_threshold`. - Dimensions, - /// LLM classifier returned a usable verdict. - #[serde(rename = "llm-classifier")] - LlmClassifier, - /// Classifier was absent or failed; picker default tier was used. - FallOpen, - /// A context-window overflow retried the configured fallback target. - ContextOverflowFallback, -} - -impl StageRouterDecisionSource { - /// Stable lowercase label used in stats JSON. - pub fn as_str(self) -> &'static str { - match self { - Self::Override => "override", - Self::Dimensions => "dimensions", - Self::LlmClassifier => "llm-classifier", - Self::FallOpen => "fall_open", - Self::ContextOverflowFallback => "context_overflow_fallback", - } - } -} - -/// StageRouter routing decision with the selected target and scorer metadata. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct StageRouterDecision { - /// Selected capable/efficient side. - pub tier: StageRouterTier, - /// Selected target id. - pub selected_target: LlmTargetId, - /// Selected upstream model. - pub selected_model: switchyard_core::ModelId, - /// Client-provided model before routing. - pub original_model: Option, - /// Decision source for observability. - pub source: StageRouterDecisionSource, - /// Linear scorer value in `[-1.0, 1.0]`. - pub score: f64, - /// Router confidence when the decision source produced one. - pub confidence: Option, -} - -impl StageRouterProfile { - async fn route_request(&self, mut input: ProfileInput) -> Result { - let signal = extract_tool_signals_with_window(&input.request, self.signal_recent_window); - let original_model = input.request.model().map(std::borrow::ToOwned::to_owned); - let decision = self.pick(&input.request, &signal, original_model).await?; - input.request.set_model(decision.selected_model.as_str()); - self.record_decision_source(decision.source)?; - Ok(StageRouterProcessedRequest { - profile_input: input, - decision, - }) - } - - // Routing decision flow: - // 1. Hard overrides choose capable for critical failures or efficient for clean tests. - // 2. Dimensions scoring chooses by score sign when confidence clears the threshold. - // 3. Low-confidence requests use the optional LLM classifier when configured. - // 4. Missing or failed classifier output falls open to the picker default tier. - async fn pick( - &self, - request: &ChatRequest, - signal: &ToolResultSignal, - original_model: Option, - ) -> Result { - if let Some(tier) = apply_overrides(signal) { - return self.decision_for_tier( - tier, - original_model, - StageRouterDecisionSource::Override, - 0.0, - Some(1.0), - ); - } - - let score = score_signal(signal); - if score.confidence >= self.confidence_threshold { - let tier = if score.score > 0.0 { - StageRouterTier::Capable - } else { - StageRouterTier::Efficient - }; - return self.decision_for_tier( - tier, - original_model, - StageRouterDecisionSource::Dimensions, - score.score, - Some(score.confidence), - ); - } - - if let Some(classifier) = &self.classifier { - if let Some(tier) = classifier - .classify(request, signal, self.stats_handle()) - .await - { - return self.decision_for_tier( - tier, - original_model, - StageRouterDecisionSource::LlmClassifier, - score.score, - None, - ); - } - } - - self.decision_for_tier( - self.picker.default_tier(), - original_model, - StageRouterDecisionSource::FallOpen, - score.score, - Some(score.confidence), - ) - } - - fn decision_for_tier( - &self, - tier: StageRouterTier, - original_model: Option, - source: StageRouterDecisionSource, - score: f64, - confidence: Option, - ) -> Result { - let backend = self.backend_for_tier(tier); - let target = backend.target(); - Ok(StageRouterDecision { - tier, - selected_target: target.id.clone(), - selected_model: target.model.clone(), - original_model, - source, - score, - confidence, - }) - } - - fn fallback_decision(&self, decision: &StageRouterDecision) -> Result { - let backend = self.backend_for_target(&self.fallback_target_on_evict)?; - let target = backend.target(); - Ok(StageRouterDecision { - tier: self.tier_for_target(&target.id)?, - selected_target: target.id.clone(), - selected_model: target.model.clone(), - original_model: decision.original_model.clone(), - source: StageRouterDecisionSource::ContextOverflowFallback, - score: decision.score, - confidence: decision.confidence, - }) - } - - fn retry_processed_request( - &self, - processed: &StageRouterProcessedRequest, - ) -> Result { - let decision = self.fallback_decision(&processed.decision)?; - let mut profile_input = processed.profile_input.clone(); - profile_input - .request - .set_model(decision.selected_model.as_str()); - Ok(StageRouterProcessedRequest { - profile_input, - decision, - }) - } - - fn backend_for_tier(&self, tier: StageRouterTier) -> &TargetBackend { - match tier { - StageRouterTier::Capable => &self.capable_backend, - StageRouterTier::Efficient => &self.efficient_backend, - } - } - - fn backend_for_target(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> { - if *target_id == self.capable_backend.target().id { - Ok(&self.capable_backend) - } else if *target_id == self.efficient_backend.target().id { - Ok(&self.efficient_backend) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "stage_router selected target {target_id} that is not configured for this profile" - ))) - } - } - - fn tier_for_target(&self, target_id: &LlmTargetId) -> Result { - if *target_id == self.capable_backend.target().id { - Ok(StageRouterTier::Capable) - } else if *target_id == self.efficient_backend.target().id { - Ok(StageRouterTier::Efficient) - } else { - Err(SwitchyardError::InvalidConfig(format!( - "stage_router target {target_id} is not configured for this profile" - ))) - } - } - - async fn call_selected( - &self, - processed: &StageRouterProcessedRequest, - ) -> (Result, f64) { - let started_at = Instant::now(); - let backend = match self.backend_for_target(&processed.decision.selected_target) { - Ok(backend) => backend, - Err(error) => return (Err(error), 0.0), - }; - let result = backend.call(&processed.profile_input.request).await; - let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0; - (result, latency_ms) - } - - fn stats_handle(&self) -> Option<&StatsAccumulator> { - self.enable_stats.then_some(&self.stats) - } - - fn record_decision_source(&self, source: StageRouterDecisionSource) -> Result<()> { - if let Some(stats) = self.stats_handle() { - stats.record_routing_decision(STAGE_ROUTER_PROFILE_TYPE, source.as_str())?; - } - Ok(()) - } - - fn record_success( - &self, - decision: &StageRouterDecision, - response: ChatResponse, - profile_started_at: Instant, - backend_latency_ms: f64, - ) -> Result { - let Some(stats) = self.stats_handle() else { - return Ok(response); - }; - stats.record_success( - decision.selected_model.as_str(), - Some(backend_latency_ms), - Some(decision.tier.as_str()), - )?; - record_usage_or_wrap_stream( - stats, - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - profile_started_at, - backend_latency_ms, - response, - ) - } - - fn record_error(&self, decision: &StageRouterDecision) -> Result<()> { - if let Some(stats) = self.stats_handle() { - stats.record_error( - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - )?; - } - Ok(()) - } - - fn routing_metadata(&self, decision: &StageRouterDecision) -> RoutingMetadata { - RoutingMetadata { - selected_model: Some(decision.selected_model.to_string()), - selected_tier: Some(decision.tier.as_str().to_string()), - confidence: decision.confidence, - router_version: Some("stage_router:v1".to_string()), - tolerance: Some(self.confidence_threshold), - rationale: Some(format!( - "stage_router source={}; score={}; selected {}", - decision.source.as_str(), - decision.score, - decision.tier.as_str() - )), - } - } -} - -#[async_trait] -impl ProfileHooks for StageRouterProfile { - type ProcessedRequest = StageRouterProcessedRequest; - - /// Extracts signals, picks a tier, and rewrites the request model. - async fn process(&self, input: ProfileInput) -> Result { - self.route_request(input).await - } - - /// Leaves the backend response unchanged after stage_router routing completes. - async fn rprocess( - &self, - _processed: &Self::ProcessedRequest, - response: ChatResponse, - ) -> Result { - Ok(response) - } -} - -#[async_trait] -impl Profile for StageRouterProfile { - /// Executes stage_router routing with one context-window fallback retry. - async fn run(&self, input: ProfileInput) -> Result { - let profile_started_at = Instant::now(); - let processed = self.process(input).await?; - let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await; - match first_result { - Ok(response) => { - let response = self.record_success( - &processed.decision, - response, - profile_started_at, - first_backend_latency_ms, - )?; - let response = self.rprocess(&processed, response).await?; - return Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(&processed.decision), - )); - } - Err(SwitchyardError::ContextWindowExceeded { .. }) => { - let retry = self.retry_processed_request(&processed)?; - let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await; - match retry_result { - Ok(response) => { - let response = self.record_success( - &retry.decision, - response, - profile_started_at, - retry_backend_latency_ms, - )?; - let response = self.rprocess(&retry, response).await?; - return Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(&retry.decision), - )); - } - Err(SwitchyardError::ContextWindowExceeded { target_id, .. }) => { - self.record_error(&retry.decision)?; - return Err(SwitchyardError::ContextPoolExhausted { - last_target_id: target_id, - reason: "all attempted targets returned context-window overflow" - .to_string(), - }); - } - Err(error) => { - self.record_error(&retry.decision)?; - return Err(error); - } - } - } - Err(error) => { - self.record_error(&processed.decision)?; - return Err(error); - } - } - } -} - -struct ScoreResult { - score: f64, - confidence: f64, -} - -struct CodingAgentDimensions { - severity: f64, - no_error_streak_intensity: f64, - write_intensity: f64, - edit_intensity: f64, - recent_write_intensity: f64, - planning_active: f64, - pure_bash_intensity: f64, - stuck_exploring: f64, - no_progress: f64, - tests_passed: f64, -} - -impl CodingAgentDimensions { - fn value(&self, name: &str) -> f64 { - match name { - "severity" => self.severity, - "no_error_streak_intensity" => self.no_error_streak_intensity, - "write_intensity" => self.write_intensity, - "edit_intensity" => self.edit_intensity, - "recent_write_intensity" => self.recent_write_intensity, - "planning_active" => self.planning_active, - "pure_bash_intensity" => self.pure_bash_intensity, - "stuck_exploring" => self.stuck_exploring, - "no_progress" => self.no_progress, - "tests_passed" => self.tests_passed, - _ => 0.0, - } - } -} - -fn score_signal(signal: &ToolResultSignal) -> ScoreResult { - let dimensions = dimensions_from_signal(signal); - let raw = DEFAULT_WEIGHTS - .iter() - .map(|(name, weight)| dimensions.value(name) * weight) - .sum::(); - let score = raw.clamp(-1.0, 1.0); - ScoreResult { - score, - confidence: score.abs(), - } -} - -fn dimensions_from_signal(signal: &ToolResultSignal) -> CodingAgentDimensions { - let total_tool_ops = signal.write_count + signal.edit_count + signal.read_count; - let recent_tool_ops = - signal.recent_write_count + signal.recent_edit_count + signal.recent_read_count; - let stuck = signal.turn_depth >= 8 && signal.write_count <= 1 && signal.read_count >= 5; - let no_progress = signal.turn_depth > 60 && signal.write_count == 0; - - CodingAgentDimensions { - severity: f64::from(signal.severity), - no_error_streak_intensity: saturating(f64::from(signal.no_error_streak), 3.0), - write_intensity: ratio(signal.write_count, total_tool_ops), - edit_intensity: ratio(signal.edit_count, total_tool_ops), - recent_write_intensity: ratio(signal.recent_write_count, recent_tool_ops), - planning_active: if signal.recent_todowrite_count > 0 { - 1.0 - } else { - 0.0 - }, - pure_bash_intensity: saturating(f64::from(signal.pure_bash_streak), PURE_BASH_NORM), - stuck_exploring: if stuck { 1.0 } else { 0.0 }, - no_progress: if no_progress { 1.0 } else { 0.0 }, - tests_passed: if signal.tests_passed && signal.write_count >= 3 { - 1.0 - } else { - 0.0 - }, - } -} - -fn apply_overrides(signal: &ToolResultSignal) -> Option { - if signal.severity >= SEVERITY_CRITICAL { - return Some(StageRouterTier::Capable); - } - if signal.tests_passed - && signal.turn_depth >= CLEAN_TESTS_MIN_TURN_DEPTH - && signal.write_count <= CLEAN_TESTS_MAX_WRITES - { - return Some(StageRouterTier::Efficient); - } - None -} - -fn saturating(value: f64, scale: f64) -> f64 { - if value <= 0.0 { - 0.0 - } else { - 1.0 - (-value / scale).exp() - } -} - -fn ratio(numerator: u32, denominator: u32) -> f64 { - if denominator == 0 { - 0.0 - } else { - f64::from(numerator) / f64::from(denominator) - } -} - -struct StageRouterTierClassifier { - config: StageRouterClassifierConfig, - client: reqwest::Client, - disable_reasoning: bool, -} - -impl StageRouterTierClassifier { - fn new(config: &StageRouterClassifierConfig) -> Result { - config.validate()?; - let client = reqwest::Client::builder() - .timeout(Duration::from_secs_f64(config.timeout_secs)) - .build() - .map_err(|error| { - SwitchyardError::InvalidConfig(format!( - "failed to build stage_router classifier HTTP client: {error}" - )) - })?; - Ok(Self { - config: config.clone(), - client, - disable_reasoning: model_accepts_reasoning_hint(config.model.as_str()), - }) - } - - async fn classify( - &self, - request: &ChatRequest, - signal: &ToolResultSignal, - stats: Option<&StatsAccumulator>, - ) -> Option { - let started_at = Instant::now(); - let response = match self - .client - .post(chat_completions_url(self.config.base_url.as_deref())) - .bearer_auth(&self.config.api_key) - .json(&self.request_body(request, signal)) - .send() - .await - { - Ok(response) => response, - Err(error) => { - tracing::warn!(error = %error, "stage_router classifier call failed; falling open"); - record_classifier_error(stats, self.config.model.as_str()); - return None; - } - }; - - if !response.status().is_success() { - tracing::warn!( - status = %response.status(), - "stage_router classifier returned error status; falling open" - ); - record_classifier_error(stats, self.config.model.as_str()); - return None; - } - - let body = match response.json::().await { - Ok(body) => body, - Err(error) => { - tracing::warn!(error = %error, "stage_router classifier returned invalid JSON; falling open"); - record_classifier_error(stats, self.config.model.as_str()); - return None; - } - }; - record_classifier_usage( - stats, - self.config.model.as_str(), - usage_from_body(&body), - started_at.elapsed().as_secs_f64() * 1000.0, - ); - let tier = parse_classifier_tier(&body); - if tier.is_none() { - record_classifier_error(stats, self.config.model.as_str()); - } - tier - } - - fn request_body(&self, request: &ChatRequest, signal: &ToolResultSignal) -> Value { - let system_prompt = self - .config - .system_prompt - .as_deref() - .unwrap_or(CLASSIFIER_SYSTEM_PROMPT); - let mut body = json!({ - "model": self.config.model, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": summarise_signal(request, signal, self.config.recent_turn_window)}, - ], - "temperature": 0, - "response_format": {"type": "json_object"}, - "max_tokens": self.config.max_tokens, - }); - if self.disable_reasoning { - body["chat_template_kwargs"] = json!({"enable_thinking": false}); - } - body - } -} - -fn record_classifier_usage( - stats: Option<&StatsAccumulator>, - model: &str, - usage: TokenUsage, - latency_ms: f64, -) { - if let Some(stats) = stats { - if let Err(error) = stats.record_classifier_usage(model, usage, Some(latency_ms)) { - tracing::debug!(error = %error, "failed to record stage_router classifier usage"); - } - } -} - -fn record_classifier_error(stats: Option<&StatsAccumulator>, model: &str) { - if let Some(stats) = stats { - if let Err(error) = stats.record_classifier_error(model) { - tracing::debug!(error = %error, "failed to record stage_router classifier error"); - } - } -} - -fn parse_classifier_tier(body: &Value) -> Option { - let content = body - .get("choices")? - .as_array()? - .first()? - .get("message")? - .get("content")? - .as_str()?; - let payload = serde_json::from_str::(content).ok()?; - match payload.get("tier").and_then(Value::as_str) { - Some("capable") => Some(StageRouterTier::Capable), - Some("efficient") => Some(StageRouterTier::Efficient), - _ => None, - } -} - -fn summarise_signal( - request: &ChatRequest, - signal: &ToolResultSignal, - recent_window: usize, -) -> String { - let state_line = format!( - "State: turn_depth={}, severity={:.1}, writes={}, edits={}, reads={}, todowrites={}, recent_writes={}, recent_edits={}, recent_reads={}, pure_bash_streak={}, no_error_streak={}, tests_passed={}", - signal.turn_depth, - signal.severity, - signal.write_count, - signal.edit_count, - signal.read_count, - signal.todowrite_count, - signal.recent_write_count, - signal.recent_edit_count, - signal.recent_read_count, - signal.pure_bash_streak, - signal.no_error_streak, - signal.tests_passed, - ); - let recent_messages = recent_messages(request, recent_window); - if recent_messages.is_empty() { - return format!("Decide CAPABLE or EFFICIENT for the next call. {state_line}"); - } - - let mut lines = vec![ - "Decide CAPABLE or EFFICIENT for the next call.".to_string(), - state_line, - "Recent turns (most recent last):".to_string(), - ]; - lines.extend(recent_messages.iter().map(format_message)); - lines.join("\n") -} - -fn recent_messages(request: &ChatRequest, recent_window: usize) -> Vec { - if recent_window == 0 { - return Vec::new(); - } - let Some(messages) = request - .body() - .as_object() - .and_then(|body| body.get("messages")) - .and_then(Value::as_array) - else { - return Vec::new(); - }; - messages - .iter() - .skip(messages.len().saturating_sub(recent_window)) - .cloned() - .collect() -} - -fn format_message(message: &Value) -> String { - let Some(object) = message.as_object() else { - let rendered = message.to_string(); - return format!("[?] {}", truncate(&rendered, 400)); - }; - let role = object.get("role").and_then(Value::as_str).unwrap_or("?"); - let body = match object.get("content") { - Some(Value::String(text)) => text.clone(), - Some(Value::Array(blocks)) => blocks - .iter() - .filter_map(format_content_block) - .collect::>() - .join(" "), - Some(other) => other.to_string(), - None => "(empty)".to_string(), - }; - format!("[{role}] {}", truncate(&body, 400)) -} - -fn format_content_block(block: &Value) -> Option { - let object = block.as_object()?; - match object.get("type").and_then(Value::as_str) { - Some("tool_use") => Some(format!( - "", - object.get("name").and_then(Value::as_str).unwrap_or("?") - )), - Some("tool_result") => Some(format!( - "", - truncate( - &object - .get("content") - .map(Value::to_string) - .unwrap_or_default(), - 120 - ) - )), - Some("text") => Some( - object - .get("text") - .and_then(Value::as_str) - .unwrap_or("") - .to_string(), - ), - _ => None, - } -} - -fn truncate(text: &str, max_chars: usize) -> String { - let mut chars = text.chars(); - let truncated = chars.by_ref().take(max_chars).collect::(); - if chars.next().is_some() { - format!("{truncated}...") - } else { - truncated - } -} - -fn chat_completions_url(base_url: Option<&str>) -> String { - format!( - "{}/chat/completions", - base_url - .unwrap_or(DEFAULT_OPENAI_BASE_URL) - .trim_end_matches('/') - ) -} - -fn model_accepts_reasoning_hint(model: &str) -> bool { - let lowered = model.to_ascii_lowercase(); - !["anthropic", "bedrock", "claude"] - .iter() - .any(|tag| lowered.contains(tag)) -} - -fn default_picker() -> StageRouterPickerMode { - StageRouterPickerMode::CapableFirst -} - -fn default_confidence_threshold() -> f64 { - DEFAULT_CONFIDENCE_THRESHOLD -} - -fn default_signal_recent_window() -> usize { - DEFAULT_RECENT_WINDOW -} - -fn default_classifier_timeout_secs() -> f64 { - DEFAULT_CLASSIFIER_TIMEOUT_SECS -} - -fn default_classifier_recent_turn_window() -> usize { - DEFAULT_CLASSIFIER_RECENT_TURN_WINDOW -} - -fn default_classifier_max_tokens() -> u32 { - CLASSIFIER_MAX_TOKENS -} - -fn default_enable_stats() -> bool { - true -} - -#[cfg(test)] -mod tests { - use std::sync::{Arc, Mutex}; - - use async_trait::async_trait; - use serde_json::json; - use switchyard_core::{BackendFormat, ModelId}; - - use crate::backend::ProfileBackend; - use crate::RequestMetadata; - - use super::*; - - #[derive(Clone, Debug, PartialEq)] - struct ObservedCall { - backend: &'static str, - body: Value, - } - - #[derive(Clone, Debug)] - enum BackendAction { - Ok, - ContextOverflow, - } - - struct TestBackend { - name: &'static str, - target_id: String, - calls: Arc>>, - actions: Mutex>, - } - - #[async_trait] - impl ProfileBackend for TestBackend { - async fn call(&self, request: &ChatRequest) -> Result { - self.calls - .lock() - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))? - .push(ObservedCall { - backend: self.name, - body: request.body().clone(), - }); - let action = self - .actions - .lock() - .map_err(|_| SwitchyardError::Other("actions mutex poisoned".to_string()))? - .pop() - .unwrap_or(BackendAction::Ok); - match action { - BackendAction::Ok => Ok(ChatResponse::openai_completion(json!({ - "served_by": self.name, - "model": request.model(), - "usage": { - "prompt_tokens": 13, - "completion_tokens": 5, - }, - }))), - BackendAction::ContextOverflow => Err(SwitchyardError::ContextWindowExceeded { - target_id: self.target_id.clone(), - model: request.model().unwrap_or("").to_string(), - message: "prompt is too long".to_string(), - }), - } - } - } - - fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) - } - - fn target_backend( - target: &LlmTarget, - name: &'static str, - calls: Arc>>, - actions: Vec, - ) -> TargetBackend { - let mut actions = actions; - actions.reverse(); - TargetBackend::new( - target.clone(), - Arc::new(TestBackend { - name, - target_id: target.id.to_string(), - calls, - actions: Mutex::new(actions), - }), - ) - } - - fn observed(calls: &Arc>>) -> Result> { - calls - .lock() - .map(|calls| calls.clone()) - .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string())) - } - - fn profile_input(request: ChatRequest) -> ProfileInput { - ProfileInput { - request, - metadata: RequestMetadata::default(), - } - } - - fn profile( - capable: LlmTarget, - efficient: LlmTarget, - picker: StageRouterPickerMode, - confidence_threshold: f64, - efficient_actions: Vec, - capable_actions: Vec, - ) -> Result<(StageRouterProfile, Arc>>)> { - let calls = Arc::new(Mutex::new(Vec::new())); - let profile = StageRouterProfile { - capable_backend: target_backend( - &capable, - "capable-backend", - calls.clone(), - capable_actions, - ), - efficient_backend: target_backend( - &efficient, - "efficient-backend", - calls.clone(), - efficient_actions, - ), - fallback_target_on_evict: capable.id.clone(), - picker, - confidence_threshold, - signal_recent_window: DEFAULT_RECENT_WINDOW, - classifier: None, - stats: StatsAccumulator::new(), - enable_stats: true, - }; - Ok((profile, calls)) - } - - #[tokio::test] - async fn stage_router_routes_critical_tool_errors_to_capable() -> Result<()> { - let (profile, calls) = profile( - target("capable", "frontier/model")?, - target("efficient", "cheap/model")?, - StageRouterPickerMode::EfficientFirst, - 0.7, - vec![BackendAction::Ok], - vec![BackendAction::Ok], - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "smart-stage-router", - "messages": [ - {"role": "assistant", "tool_calls": [{ - "type": "function", - "function": {"name": "Bash", "arguments": "{\"command\":\"python test.py\"}"} - }]}, - {"role": "tool", "tool_call_id": "call_1", "content": "Out of memory"} - ], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable")); - assert_eq!(routing_metadata.confidence, Some(1.0)); - assert_eq!( - routing_metadata.router_version.as_deref(), - Some("stage_router:v1") - ); - let response = response.response; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].backend, "capable-backend"); - assert_eq!(calls[0].body["model"], "frontier/model"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["served_by"], "capable-backend"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - - let snapshot = profile.stats.snapshot()?; - assert_eq!( - snapshot - .routing_decisions - .get("stage_router") - .and_then(|sources| sources.get("override")), - Some(&1) - ); - assert_eq!( - snapshot - .models - .get("frontier/model") - .and_then(|model| model.tier.as_deref()), - Some("capable") - ); - Ok(()) - } - - #[tokio::test] - async fn stage_router_threshold_zero_accepts_neutral_scorer_as_efficient() -> Result<()> { - let (profile, calls) = profile( - target("capable", "frontier/model")?, - target("efficient", "cheap/model")?, - StageRouterPickerMode::CapableFirst, - 0.0, - vec![BackendAction::Ok], - vec![BackendAction::Ok], - )?; - - let processed = profile - .process(profile_input(ChatRequest::openai_chat(json!({ - "model": "smart-stage-router", - "messages": [{"role": "user", "content": "continue"}], - })))) - .await?; - - assert_eq!(processed.decision.tier, StageRouterTier::Efficient); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::Dimensions - ); - assert_eq!(processed.profile_input.request.model(), Some("cheap/model")); - assert!(observed(&calls)?.is_empty()); - Ok(()) - } - - #[tokio::test] - async fn stage_router_retries_configured_fallback_after_context_overflow() -> Result<()> { - let (profile, calls) = profile( - target("capable", "frontier/model")?, - target("efficient", "cheap/model")?, - StageRouterPickerMode::EfficientFirst, - 0.7, - vec![BackendAction::ContextOverflow], - vec![BackendAction::Ok], - )?; - - let response = profile - .run(profile_input(ChatRequest::openai_chat(json!({ - "model": "smart-stage-router", - "messages": [{"role": "user", "content": "continue"}], - })))) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable")); - assert!(routing_metadata - .rationale - .as_deref() - .is_some_and(|reason| reason.contains("source=context_overflow_fallback"))); - let response = response.response; - - let calls = observed(&calls)?; - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].backend, "efficient-backend"); - assert_eq!(calls[0].body["model"], "cheap/model"); - assert_eq!(calls[1].backend, "capable-backend"); - assert_eq!(calls[1].body["model"], "frontier/model"); - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["served_by"], "capable-backend"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - - let snapshot = profile.stats.snapshot()?; - assert_eq!(snapshot.total_requests, 1); - assert_eq!( - snapshot - .models - .get("frontier/model") - .and_then(|model| model.tier.as_deref()), - Some("capable") - ); - assert_eq!( - snapshot - .routing_decisions - .get("stage_router") - .and_then(|sources| sources.get("fall_open")), - Some(&1) - ); - Ok(()) - } - - #[test] - fn stage_router_config_rejects_unknown_fallback_target() -> Result<()> { - let config = StageRouterProfileConfig { - capable: target("capable", "frontier/model")?, - efficient: target("efficient", "cheap/model")?, - fallback_target_on_evict: LlmTargetId::new("ghost")?, - picker: StageRouterPickerMode::CapableFirst, - confidence_threshold: 0.7, - signal_recent_window: DEFAULT_RECENT_WINDOW, - classifier: None, - enable_stats: true, - }; - - let error = config - .validate() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected validation error".to_string()); - assert!(error.contains("fallback_target_on_evict")); - Ok(()) - } - - #[test] - fn classifier_request_uses_reasoning_hint_for_vllm_models() -> Result<()> { - let classifier = StageRouterTierClassifier::new(&StageRouterClassifierConfig { - model: "nvidia/deepseek-ai/deepseek-v4-flash".to_string(), - api_key: "test-key".to_string(), - base_url: None, - timeout_secs: 1.0, - recent_turn_window: 2, - max_tokens: CLASSIFIER_MAX_TOKENS, - system_prompt: None, - })?; - - let body = classifier.request_body( - &ChatRequest::openai_chat(json!({ - "model": "smart-stage-router", - "messages": [{"role": "user", "content": "hi"}], - })), - &ToolResultSignal::default(), - ); - - assert_eq!( - body["chat_template_kwargs"], - json!({"enable_thinking": false}) - ); - assert_eq!(body["max_tokens"], CLASSIFIER_MAX_TOKENS); - Ok(()) - } -} diff --git a/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md b/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md deleted file mode 100644 index 25f570fc..00000000 --- a/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md +++ /dev/null @@ -1,5 +0,0 @@ -You are a routing classifier inside an agentic coding stage-router. Given a compact summary of the agent's recent tool activity, decide whether the next model call should go to the CAPABLE tier (frontier, expensive) or the EFFICIENT tier (cheap, less powerful). - -Respond with strict JSON: {"tier": "capable"} or {"tier": "efficient"}. - -Pick EFFICIENT when the agent shows concrete, low-friction progress (writes landing, tests passing, edits without errors). Pick CAPABLE when the agent is stalled, hitting errors, or facing a task likely to require careful reasoning. diff --git a/crates/switchyard-components-v2/src/profiles/subagent_override.rs b/crates/switchyard-components-v2/src/profiles/subagent_override.rs deleted file mode 100644 index b9275b4b..00000000 --- a/crates/switchyard-components-v2/src/profiles/subagent_override.rs +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Sub-agent override combinator for v2 profiles. -//! -//! Wraps any profile without changing its behavior for normal traffic. A -//! request whose headers mark delegated sub-agent work -//! ([`switchyard_protocol::Metadata::is_subagent_work`]) is served directly by -//! a fixed worker target — keeping a sub-agent loop on an intentional, -//! cache-compatible target — while every other request delegates to the wrapped -//! profile. A worker failure surfaces as a normal target error and is never -//! silently re-routed through the wrapped profile. - -use std::collections::BTreeMap; -use std::time::Instant; - -use async_trait::async_trait; -use switchyard_components::StatsAccumulator; -use switchyard_core::{LlmTarget, Result}; - -use crate::backend::{native_target_backend, TargetBackend}; -use crate::profile_stats_accumulator; -use crate::stats_recording::record_usage_or_wrap_stream; -use crate::{Profile, ProfileInput, ProfileResponse}; - -/// Wraps a profile, routing delegated sub-agent work to a fixed worker target. -pub(crate) struct SubagentOverrideProfile { - inner: Box, - worker: TargetBackend, - stats: StatsAccumulator, -} - -impl SubagentOverrideProfile { - /// Wraps `inner`, routing recognized sub-agent work requests to `worker`. - pub(crate) fn new(inner: Box, target: LlmTarget) -> Result { - Ok(Self { - inner, - worker: native_target_backend(target)?, - stats: profile_stats_accumulator(), - }) - } -} - -/// Returns true when the request headers signal delegated sub-agent work. -/// -/// Normalizes multi-value headers to single-value (first occurrence wins) -/// before calling the protocol-layer detection logic. -fn is_subagent_work(headers: &BTreeMap>) -> bool { - let flat: BTreeMap = headers - .iter() - .filter_map(|(k, vs)| vs.first().map(|v| (k.clone(), v.clone()))) - .collect(); - switchyard_protocol::Metadata::from_headers(&flat).is_subagent_work() -} - -#[async_trait] -impl Profile for SubagentOverrideProfile { - /// Routes sub-agent work to the worker target; delegates everything else. - async fn run(&self, mut input: ProfileInput) -> Result { - if !is_subagent_work(&input.metadata.headers) { - return self.inner.run(input).await; - } - - let profile_started_at = Instant::now(); - let target_model = self.worker.target().model.clone(); - input.request.set_model(target_model.as_str()); - - let backend_started_at = Instant::now(); - let response = match self.worker.call(&input.request).await { - Ok(response) => response, - Err(error) => { - self.stats.record_error(target_model.as_str(), None)?; - return Err(error); - } - }; - - let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; - self.stats - .record_success(target_model.to_string(), Some(backend_latency_ms), None)?; - let response = record_usage_or_wrap_stream( - &self.stats, - target_model.as_str(), - None, - profile_started_at, - backend_latency_ms, - response, - )?; - Ok(ProfileResponse::from(response)) - } -} diff --git a/crates/switchyard-components-v2/src/stats.rs b/crates/switchyard-components-v2/src/stats.rs deleted file mode 100644 index 3c245b51..00000000 --- a/crates/switchyard-components-v2/src/stats.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared stats surface for profile-owned runtimes. - -use std::sync::OnceLock; - -use switchyard_components::StatsAccumulator; - -/// Returns the process-wide stats accumulator used by v2 profiles. -pub fn profile_stats_accumulator() -> StatsAccumulator { - static PROFILE_STATS: OnceLock = OnceLock::new(); - PROFILE_STATS.get_or_init(StatsAccumulator::new).clone() -} diff --git a/crates/switchyard-components-v2/src/stats_recording.rs b/crates/switchyard-components-v2/src/stats_recording.rs deleted file mode 100644 index 501d30b4..00000000 --- a/crates/switchyard-components-v2/src/stats_recording.rs +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Records provider token usage for buffered and streaming responses. -//! -//! Buffered responses carry usage in the body and can be recorded synchronously. -//! Streaming responses only emit usage inside a later event, so the response -//! stream is wrapped and usage is recorded when the first usage-bearing event -//! flows through — no buffering, no added latency, and the client still sees -//! every event. - -use std::time::Instant; - -use async_stream::try_stream; -use futures_util::StreamExt; -use switchyard_components::stats::{ - openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, usage_from_body, - AnthropicStreamUsage, StatsAccumulator, TokenUsage, -}; -use switchyard_core::{BoxResponseStream, ChatResponse, Result}; - -/// Records usage from a buffered response, or wraps a streaming response so -/// usage is recorded when its usage-bearing event arrives. -pub(crate) fn record_usage_or_wrap_stream( - stats: &StatsAccumulator, - model: &str, - tier: Option<&str>, - profile_started_at: Instant, - backend_latency_ms: f64, - response: ChatResponse, -) -> Result { - // We report time-to-first-token as latency. Otherwise in streaming case we'd be including - // the client's processing time. - let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0; - match response { - ChatResponse::OpenAiCompletion(_) - | ChatResponse::OpenAiResponsesCompletion(_) - | ChatResponse::AnthropicCompletion(_) => { - let usage = response.body().map(usage_from_body).unwrap_or_default(); - record_usage( - stats, - model, - tier, - total_latency_ms, - backend_latency_ms, - usage, - )?; - Ok(response) - } - ChatResponse::OpenAiStream(stream) => Ok(ChatResponse::OpenAiStream(wrap_openai_chat( - stream, - stats.clone(), - model.to_string(), - tier.map(str::to_string), - total_latency_ms, - backend_latency_ms, - ))), - ChatResponse::OpenAiResponsesStream(stream) => { - Ok(ChatResponse::OpenAiResponsesStream(wrap_openai_responses( - stream, - stats.clone(), - model.to_string(), - tier.map(str::to_string), - total_latency_ms, - backend_latency_ms, - ))) - } - ChatResponse::AnthropicStream(stream) => Ok(ChatResponse::AnthropicStream(wrap_anthropic( - stream, - stats.clone(), - model.to_string(), - tier.map(str::to_string), - total_latency_ms, - backend_latency_ms, - ))), - } -} - -/// Wraps an OpenAI chat completions stream to record usage stats. -/// -/// Usage is typically only present in the last JSON SSE chunk, but we only -/// record the first instance to avoid double counting in case of middle-box weirdness. -/// -/// Pass-through, no buffering or dropping. -fn wrap_openai_chat( - mut stream: BoxResponseStream, - stats: StatsAccumulator, - model: String, - tier: Option, - total_latency_ms: f64, - backend_latency_ms: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut committed = false; - while let Some(event) = stream.next().await { - let event = event?; - if !committed { - if let Some(usage) = openai_chat_usage_from_stream_event(&event) { - log_stream_record_result( - record_usage( - &stats, - &model, - tier.as_deref(), - total_latency_ms, - backend_latency_ms, - usage, - ), - &model, - ); - committed = true; - } - } - yield event; - } - }) -} - -/// Wraps an OpenAI responses stream to record usage stats. -fn wrap_openai_responses( - mut stream: BoxResponseStream, - stats: StatsAccumulator, - model: String, - tier: Option, - total_latency_ms: f64, - backend_latency_ms: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut committed = false; - while let Some(event) = stream.next().await { - let event = event?; - if !committed { - if let Some(usage) = openai_responses_usage_from_stream_event(&event) { - log_stream_record_result( - record_usage( - &stats, - &model, - tier.as_deref(), - total_latency_ms, - backend_latency_ms, - usage, - ), - &model, - ); - committed = true; - } - } - yield event; - } - }) -} - -/// Wraps an Anthropic messages stream to record usage stats. -fn wrap_anthropic( - mut stream: BoxResponseStream, - stats: StatsAccumulator, - model: String, - tier: Option, - total_latency_ms: f64, - backend_latency_ms: f64, -) -> BoxResponseStream { - Box::pin(try_stream! { - let mut stream_usage = AnthropicStreamUsage::default(); - while let Some(event) = stream.next().await { - let event = event?; - if let Some(usage) = stream_usage.observe(&event) { - log_stream_record_result( - record_usage( - &stats, - &model, - tier.as_deref(), - total_latency_ms, - backend_latency_ms, - usage, - ), - &model, - ); - } - yield event; - } - }) -} - -fn record_usage( - stats: &StatsAccumulator, - model: &str, - tier: Option<&str>, - total_latency_ms: f64, - backend_latency_ms: f64, - usage: TokenUsage, -) -> Result<()> { - let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0); - stats.record_usage_after_success_attribution( - model.to_string(), - usage, - Some(total_latency_ms), - Some(routing_overhead_ms), - tier, - ) -} - -fn log_stream_record_result(result: Result<()>, model: &str) { - if let Err(error) = result { - tracing::warn!( - error = %error, - model = %model, - "failed to record stream usage", - ); - } -} diff --git a/crates/switchyard-components-v2/tests/config.rs b/crates/switchyard-components-v2/tests/config.rs deleted file mode 100644 index 943a34cf..00000000 --- a/crates/switchyard-components-v2/tests/config.rs +++ /dev/null @@ -1,795 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Adversarial tests for profile config parsing and profile-owned resolution. - -use serde_json::json; -use switchyard_components_v2::{ - parse_profile_config_str_with_env_lookup, NoopProfileConfig, ProfileConfig, - ProfileConfigDocument, ProfileConfigFormat, ProfileHooks, ProfileInput, RequestMetadata, -}; -use switchyard_core::{ChatRequest, ChatResponse, LlmTargetId, ProfileId, Result, SwitchyardError}; - -const YAML_CONFIG: &str = r#" -endpoints: - nvidia: - api_key: ${NVIDIA_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - timeout_secs: 120.0 - -targets: - strong: - endpoint: nvidia - model: nvidia/moonshotai/kimi-k2.5 - format: openai - weak: - endpoint: nvidia - model: nvidia/nvidia/nemotron-nano-9b-v2 - format: openai - direct-weak: - endpoint: nvidia - model: nvidia/nvidia/nemotron-nano-9b-v2 - format: openai - classifier: - endpoint: nvidia - model: nvidia/nvidia/nemotron-nano-9b-v2 - format: openai - -profiles: - direct: - type: passthrough - target: direct-weak - smart-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.7 - health-aware: - type: latency-service - latency_service_url: http://latency.local - targets: [strong, weak] - llm: - type: llm-routing - strong: strong - weak: weak - classifier: classifier - profile_name: coding_agent - stage_router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - classifier: - model: nvidia/nvidia/nemotron-nano-9b-v2 - api_key: ${NVIDIA_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - bench: - type: noop -"#; - -// Parses YAML profile config using the test environment lookup. -fn parse_yaml(input: &str) -> Result { - parse_profile_config_str_with_env_lookup(input, ProfileConfigFormat::Yaml, test_env) -} - -// Parses a profile config in the requested format using the test environment lookup. -fn parse_with_format(input: &str, format: ProfileConfigFormat) -> Result { - parse_profile_config_str_with_env_lookup(input, format, test_env) -} - -// Supplies deterministic values for environment interpolation tests. -fn test_env(name: &str) -> Option { - match name { - "NVIDIA_API_KEY" => Some("nvapi-test".to_string()), - "TENANT" => Some("team-a".to_string()), - _ => None, - } -} - -// Resolving a full YAML config should inherit endpoints and validate profile-owned configs. -#[test] -fn yaml_config_resolves_endpoints_targets_and_profile_owned_configs() -> Result<()> { - let config = parse_yaml(YAML_CONFIG)?; - let plan = config.resolve()?; - - assert_eq!(plan.target_count(), 4); - let direct_weak = plan - .target(&LlmTargetId::new("direct-weak")?) - .ok_or_else(|| SwitchyardError::Other("direct-weak target missing".to_string()))?; - assert_eq!(direct_weak.endpoint.api_key.as_deref(), Some("nvapi-test")); - assert_eq!( - direct_weak.endpoint.base_url.as_deref(), - Some("https://inference-api.nvidia.com/v1") - ); - - assert_eq!( - plan.profile_type(&ProfileId::new("direct")?), - Some("passthrough") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("smart-stage-router")?), - Some("stage_router") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("health-aware")?), - Some("latency-service") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("llm")?), - Some("llm-routing") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("stage_router")?), - Some("stage_router") - ); - - let profiles = plan.build_profiles()?; - assert_eq!(profiles.len(), 6); - - let targets = plan - .targets() - .map(|(target_id, _target)| target_id.clone()) - .collect::>(); - assert_eq!( - targets, - vec![ - LlmTargetId::new("classifier")?, - LlmTargetId::new("direct-weak")?, - LlmTargetId::new("strong")?, - LlmTargetId::new("weak")?, - ] - ); - Ok(()) -} - -// JSON and TOML should normalize into equivalent resolved config plans. -#[test] -fn json_and_toml_parse_to_the_same_document_shape() -> Result<()> { - let json_config = r#" -{ - "endpoints": { - "nvidia": { - "api_key": "${NVIDIA_API_KEY}", - "base_url": "https://inference-api.nvidia.com/v1", - "timeout_secs": 120.0 - } - }, - "targets": { - "weak": { - "endpoint": "nvidia", - "model": "nvidia/nvidia/nemotron-nano-9b-v2", - "format": "openai" - } - }, - "profiles": { - "direct": { - "type": "passthrough", - "target": "weak" - } - } -} -"#; - let toml_config = r#" -[endpoints.nvidia] -api_key = "${NVIDIA_API_KEY}" -base_url = "https://inference-api.nvidia.com/v1" -timeout_secs = 120.0 - -[targets.weak] -endpoint = "nvidia" -model = "nvidia/nvidia/nemotron-nano-9b-v2" -format = "openai" - -[profiles.direct] -type = "passthrough" -target = "weak" -"#; - - let json = parse_with_format(json_config, ProfileConfigFormat::Json)?.resolve()?; - let toml = parse_with_format(toml_config, ProfileConfigFormat::Toml)?.resolve()?; - - assert_eq!(json, toml); - Ok(()) -} - -// Unknown fields inside a profile body should be rejected by that profile's config parser. -#[test] -fn unknown_profile_fields_are_rejected_by_owning_profile_config() -> Result<()> { - let input = r#" -targets: - strong: - model: strong/model - format: openai - weak: - model: weak/model - format: openai -profiles: - bad: - type: random-routing - strong: strong - weak: weak - strong_probability: 0.3 - tina: codes well -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("unknown field")); - assert!(error.contains("tina")); - assert!(error.contains("profile bad")); - Ok(()) -} - -// Target-level `expose` stays rejected because every v2 target is addressable. -#[test] -fn target_expose_field_is_rejected_because_all_targets_are_exposed() { - let input = r#" -targets: - weak: - model: weak/model - format: openai - expose: true -profiles: - direct: - type: passthrough - target: weak -"#; - - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("unknown field")); - assert!(error.contains("expose")); -} - -// Target IDs should come only from map keys, not duplicated `id` fields. -#[test] -fn target_id_field_is_rejected_because_map_key_is_the_id() { - let input = r#" -targets: - weak: - id: duplicate - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak -"#; - - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("unknown field")); - assert!(error.contains("id")); -} - -// The top-level schema should reject stale route/table-era sections. -#[test] -fn unknown_top_level_fields_are_rejected() { - let input = r#" -endpoints: {} -targets: {} -profiles: {} -routes: {} -"#; - - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("unknown field")); - assert!(error.contains("routes")); -} - -// Unknown profile types should parse as documents but fail during profile resolution. -#[test] -fn unknown_profile_type_is_rejected_during_resolution() -> Result<()> { - let input = r#" -profiles: - bad: - type: handmade -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("handmade")); - assert!(error.contains("profile bad")); - Ok(()) -} - -// `-` and `_` are equivalent in the `type` discriminator, so a name written with -// either separator resolves to the same v2 profile: a legacy `random_routing` -// spelling, and `stage-router` for the underscore-canonical stage router. -#[test] -fn profile_type_separators_are_normalized_during_resolution() -> Result<()> { - let input = r#" -endpoints: - nvidia: - api_key: ${NVIDIA_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - -targets: - strong: - endpoint: nvidia - model: nvidia/moonshotai/kimi-k2.5 - format: openai - weak: - endpoint: nvidia - model: nvidia/nvidia/nemotron-nano-9b-v2 - format: openai - classifier: - endpoint: nvidia - model: nvidia/nvidia/nemotron-nano-9b-v2 - format: openai - -profiles: - snake-llm: - type: llm_routing - strong: strong - weak: weak - classifier: classifier - profile_name: coding_agent - hyphen-stage: - type: stage-router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.7 - snake-latency: - type: latency_service - latency_service_url: http://latency.local - targets: [strong, weak] -"#; - - let plan = parse_yaml(input)?.resolve()?; - - // Resolved profiles report the canonical type name regardless of the - // separator spelled in the config. - assert_eq!( - plan.profile_type(&ProfileId::new("snake-llm")?), - Some("llm-routing") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("hyphen-stage")?), - Some("stage_router") - ); - assert_eq!( - plan.profile_type(&ProfileId::new("snake-latency")?), - Some("latency-service") - ); - - // The profiles build, not just parse. - assert_eq!(plan.build_profiles()?.len(), 3); - Ok(()) -} - -// Missing profile type discriminators should fail while parsing the document. -#[test] -fn missing_profile_type_is_rejected_during_document_parse() { - let input = r#" -profiles: - bad: - target: weak -"#; - - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("type")); -} - -// Empty profile type discriminators should fail while parsing the document. -#[test] -fn empty_profile_type_is_rejected_during_document_parse() { - let input = r#" -profiles: - bad: - type: " " -"#; - - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("must not be empty")); -} - -// Missing environment variables should fail before document deserialization completes. -#[test] -fn missing_env_var_is_rejected() { - let var_name = format!( - "SWITCHYARD_COMPONENTS_V2_MISSING_ENV_{}", - std::process::id() - ); - let input = format!( - r#" -endpoints: - missing: - api_key: ${{{var_name}}} -"# - ); - - let error = - parse_profile_config_str_with_env_lookup(&input, ProfileConfigFormat::Yaml, test_env) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - - assert!(error.contains(&var_name)); - assert!(error.contains("not set")); -} - -// Targets that reference a missing endpoint should fail during target resolution. -#[test] -fn missing_endpoint_reference_is_rejected_during_resolution() -> Result<()> { - let input = r#" -targets: - weak: - endpoint: missing - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("unknown endpoint missing")); - Ok(()) -} - -// Macro-resolved profile target fields should fail when they name an unknown target. -#[test] -fn missing_profile_target_reference_is_rejected_by_macro_generated_resolver() -> Result<()> { - let input = r#" -targets: - weak: - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: typo -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("unknown target typo")); - assert!(error.contains("profile direct")); - Ok(()) -} - -// Target-local endpoint fields should override shared endpoints and interpolate nested strings. -#[test] -fn target_overrides_endpoint_fields_and_interpolates_nested_strings() -> Result<()> { - let input = r#" -endpoints: - nvidia: - api_key: ${NVIDIA_API_KEY} - base_url: https://inference-api.nvidia.com/v1 - timeout_secs: 120.0 -targets: - weak: - endpoint: nvidia - model: weak/model - format: openai - base_url: https://override.example/v1 - timeout_secs: 30.0 - extra_body: - metadata: - tenant: ${TENANT} -profiles: - direct: - type: passthrough - target: weak -"#; - let plan = parse_yaml(input)?.resolve()?; - let weak = plan - .target(&LlmTargetId::new("weak")?) - .ok_or_else(|| SwitchyardError::Other("weak target missing".to_string()))?; - - assert_eq!( - weak.endpoint.base_url.as_deref(), - Some("https://override.example/v1") - ); - assert_eq!(weak.endpoint.timeout_secs, Some(30.0)); - assert_eq!(weak.endpoint.api_key.as_deref(), Some("nvapi-test")); - assert_eq!( - weak.extra_body.as_ref().and_then(|body| { - body.get("metadata") - .and_then(|metadata| metadata.get("tenant")) - .and_then(serde_json::Value::as_str) - }), - Some("team-a") - ); - Ok(()) -} - -// Debug output for resolved plans should not include inherited endpoint API keys. -#[test] -fn profile_config_plan_debug_redacts_target_api_keys() -> Result<()> { - let plan = parse_yaml(YAML_CONFIG)?.resolve()?; - let debug = format!("{plan:?}"); - - assert!(!debug.contains("nvapi-test")); - assert!(debug.contains("direct-weak")); - assert!(debug.contains("passthrough")); - Ok(()) -} - -// Resolved plans should build one profile or all profiles into runtime objects. -#[test] -fn profile_config_plan_builds_runtime_profiles() -> Result<()> { - let plan = parse_yaml(YAML_CONFIG)?.resolve()?; - - let direct = plan.build_profile(&ProfileId::new("direct")?)?; - let profiles = plan.build_profiles()?; - - drop(direct); - assert_eq!(profiles.len(), 6); - assert!(profiles.contains_key(&ProfileId::new("direct")?)); - assert!(profiles.contains_key(&ProfileId::new("smart-stage-router")?)); - assert!(profiles.contains_key(&ProfileId::new("health-aware")?)); - assert!(profiles.contains_key(&ProfileId::new("llm")?)); - assert!(profiles.contains_key(&ProfileId::new("stage_router")?)); - assert!(profiles.contains_key(&ProfileId::new("bench")?)); - Ok(()) -} - -// Components-v2 stage_router config should reject route-era/top-level classifier -// knobs instead of carrying compatibility behavior forward. -#[test] -fn stage_router_rejects_top_level_classifier_knobs() -> Result<()> { - let input = r#" -targets: - strong: - model: strong/model - format: openai - weak: - model: weak/model - format: openai -profiles: - stale: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - classifier_max_tokens: 64 -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("unknown field")); - assert!(error.contains("classifier_max_tokens")); - Ok(()) -} - -// Invalid stage_router picker names fail during profile config resolution. -#[test] -fn stage_router_resolve_rejects_unknown_picker() -> Result<()> { - let input = r#" -targets: - strong: - model: strong/model - format: openai - weak: - model: weak/model - format: openai -profiles: - bad: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: not-a-picker -"#; - - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("unknown variant")); - assert!(error.contains("not-a-picker")); - Ok(()) -} - -// Strict OpenAI classifier calls require OpenAI-format classifier targets. -#[test] -fn llm_routing_build_rejects_non_openai_classifier_target() -> Result<()> { - let input = r#" -targets: - strong: - model: strong/model - format: openai - weak: - model: weak/model - format: openai - classifier: - model: classifier/model - format: anthropic -profiles: - bad: - type: llm-routing - strong: strong - weak: weak - classifier: classifier -"#; - - let error = parse_yaml(input)? - .resolve()? - .build_profile(&ProfileId::new("bad")?) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected build failure".to_string()); - assert!(error.contains("classifier target must use format: openai")); - Ok(()) -} - -// Config-owned `build()` should return a concrete profile runtime with hook methods. -#[tokio::test] -async fn profile_config_build_returns_profile_with_request_and_response_hooks() -> Result<()> { - let profile = NoopProfileConfig {}.build()?; - let input = ProfileInput { - request: ChatRequest::openai_chat(json!({ - "model": "unit/noop", - "messages": [], - })), - metadata: RequestMetadata::default(), - }; - - let processed = profile.process(input).await?; - let response = profile - .rprocess( - &processed, - ChatResponse::openai_completion(json!({ - "id": "unit-response", - "object": "chat.completion", - "model": "unit/noop", - "choices": [], - })), - ) - .await?; - - assert_eq!(processed.request.model(), Some("unit/noop")); - assert_eq!( - response - .body() - .and_then(|body| body.get("id")) - .and_then(serde_json::Value::as_str), - Some("unit-response") - ); - Ok(()) -} - -// File format detection should reject unsupported config file extensions. -#[test] -fn profile_config_format_rejects_unknown_extension() { - let error = ProfileConfigFormat::from_path("profiles.ini") - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected extension failure".to_string()); - assert!(error.contains("unsupported profile config extension")); -} - -// The envelope-level `subagent_target` is consumed before profile-owned parsing, -// so profile types that deny unknown fields still resolve, and the reference -// stays readable from the parsed document. -#[test] -fn subagent_target_envelope_field_is_stripped_and_exposed() -> Result<()> { - let input = r#" -targets: - weak: - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak - subagent_target: weak -"#; - let config = parse_yaml(input)?; - let profile_id = ProfileId::new("direct")?; - assert_eq!( - config.profile_subagent_target(&profile_id), - Some(&LlmTargetId::new("weak")?) - ); - let body_has_field = config - .profile_body(&profile_id) - .and_then(|body| body.get("subagent_target")) - .is_some(); - assert!(!body_has_field); - config.resolve()?; - Ok(()) -} - -// A profile without the envelope field reports no sub-agent target. -#[test] -fn subagent_target_is_absent_by_default() -> Result<()> { - let input = r#" -targets: - weak: - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak -"#; - let config = parse_yaml(input)?; - assert_eq!( - config.profile_subagent_target(&ProfileId::new("direct")?), - None - ); - Ok(()) -} - -// An unknown `subagent_target` reference is a startup configuration error. -#[test] -fn subagent_target_referencing_unknown_target_is_rejected_at_resolve() -> Result<()> { - let input = r#" -targets: - weak: - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak - subagent_target: ghost -"#; - let error = parse_yaml(input)? - .resolve() - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected resolve failure".to_string()); - assert!(error.contains("profile direct")); - assert!(error.contains("subagent_target references unknown target ghost")); - Ok(()) -} - -// The envelope field must be a target id string, not a structured value. -#[test] -fn subagent_target_must_be_a_target_id_string() { - let input = r#" -targets: - weak: - model: weak/model - format: openai -profiles: - direct: - type: passthrough - target: weak - subagent_target: [weak] -"#; - let error = parse_yaml(input) - .err() - .map(|error| error.to_string()) - .unwrap_or_else(|| "expected parse failure".to_string()); - assert!(error.contains("`subagent_target` must be a target id string")); -} diff --git a/crates/switchyard-components-v2/tests/latency_service_profile.rs b/crates/switchyard-components-v2/tests/latency_service_profile.rs deleted file mode 100644 index 13683129..00000000 --- a/crates/switchyard-components-v2/tests/latency_service_profile.rs +++ /dev/null @@ -1,453 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Public API tests for the components-v2 latency-service profile config. - -use std::collections::BTreeMap; -use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener, TcpStream}; -use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; - -use serde_json::{json, Value}; -use switchyard_components_v2::{ - profile_stats_accumulator, LatencyServiceProfileConfig, Profile, ProfileConfig, ProfileInput, - RequestMetadata, -}; -use switchyard_core::{ - BackendFormat, ChatRequest, ChatResponse, EndpointConfig, LlmTarget, LlmTargetId, ModelId, - Result, SwitchyardError, -}; - -fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) -} - -fn config(targets: Vec) -> LatencyServiceProfileConfig { - LatencyServiceProfileConfig { - latency_service_url: "http://latency.test".to_string(), - targets, - poll_timeout_secs: 5.0, - max_retries: 2, - } -} - -fn openai_target(id: &'static str, model: &'static str, base_url: &str) -> LlmTarget { - LlmTarget { - id: LlmTargetId::from_static(id), - model: ModelId::from_static(model), - format: BackendFormat::OpenAi, - endpoint: EndpointConfig { - base_url: Some(base_url.to_string()), - api_key: Some("test-key".to_string()), - timeout_secs: Some(5.0), - }, - extra_body: None, - extra_headers: BTreeMap::new(), - } -} - -#[derive(Clone, Debug, Default, PartialEq)] -struct MockCounts { - requests: Vec, - health: u64, - fast: u64, - slow: u64, - fast_body: Option, - slow_body: Option, -} - -#[derive(Clone, Debug, PartialEq)] -struct HttpRequest { - method: String, - path: String, - body: Value, -} - -struct MockLatencyOpenAiServer { - base_url: String, - address: SocketAddr, - receiver: Receiver>, - handle: Option>, -} - -impl MockLatencyOpenAiServer { - fn spawn(requests: usize) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?; - let address = listener - .local_addr() - .map_err(|error| SwitchyardError::Other(format!("local addr failed: {error}")))?; - let (sender, receiver) = mpsc::channel(); - let handle = thread::spawn(move || { - let result = run_mock_server(listener, requests); - let _ignored = sender.send(result); - }); - Ok(Self { - base_url: format!("http://{address}"), - address, - receiver, - handle: Some(handle), - }) - } - - fn base_url(&self) -> String { - self.base_url.clone() - } - - fn finish(&mut self) -> Result { - let result = match self.receiver.recv_timeout(Duration::from_secs(5)) { - Ok(result) => result, - Err(RecvTimeoutError::Timeout) => { - self.wake(); - self.receiver - .recv_timeout(Duration::from_secs(1)) - .map_err(|error| { - SwitchyardError::Other(format!("mock server did not finish: {error}")) - })? - } - Err(RecvTimeoutError::Disconnected) => { - Err(SwitchyardError::Other("mock server disconnected".into())) - } - }; - if let Some(handle) = self.handle.take() { - handle - .join() - .map_err(|_| SwitchyardError::Other("mock server thread panicked".into()))?; - } - result - } - - fn wake(&self) { - let _ignored = TcpStream::connect(self.address); - } -} - -impl Drop for MockLatencyOpenAiServer { - fn drop(&mut self) { - if self.handle.is_some() { - self.wake(); - } - if let Some(handle) = self.handle.take() { - let _ignored = handle.join(); - } - } -} - -fn run_mock_server(listener: TcpListener, requests: usize) -> Result { - let mut counts = MockCounts::default(); - for _ in 0..requests { - let (mut stream, _address) = listener - .accept() - .map_err(|error| SwitchyardError::Other(format!("accept failed: {error}")))?; - stream - .set_read_timeout(Some(Duration::from_secs(5))) - .map_err(|error| SwitchyardError::Other(format!("set read timeout failed: {error}")))?; - stream - .set_write_timeout(Some(Duration::from_secs(5))) - .map_err(|error| { - SwitchyardError::Other(format!("set write timeout failed: {error}")) - })?; - let request = read_http_request(&mut stream)?; - let body = response_for_request(request, &mut counts)?; - write_json_response(&mut stream, 200, body)?; - } - Ok(counts) -} - -fn read_http_request(stream: &mut TcpStream) -> Result { - let mut bytes = Vec::new(); - let mut chunk = [0_u8; 4096]; - loop { - let read = stream - .read(&mut chunk) - .map_err(|error| SwitchyardError::Other(format!("read request failed: {error}")))?; - if read == 0 { - break; - } - bytes.extend_from_slice(&chunk[..read]); - if let Some((header_end, content_length)) = request_shape(&bytes) { - let body_start = header_end + 4; - if bytes.len() >= body_start + content_length { - break; - } - } - } - - let (header_end, content_length) = request_shape(&bytes) - .ok_or_else(|| SwitchyardError::Other("request should contain headers".into()))?; - let header_text = std::str::from_utf8(&bytes[..header_end]) - .map_err(|error| SwitchyardError::Other(format!("headers should be UTF-8: {error}")))?; - let body_start = header_end + 4; - let body_end = body_start + content_length; - - let mut lines = header_text.lines(); - let first_line = lines - .next() - .ok_or_else(|| SwitchyardError::Other("missing request line".into()))?; - let mut parts = first_line.split_whitespace(); - let method = parts - .next() - .ok_or_else(|| SwitchyardError::Other("missing request method".into()))? - .to_string(); - let path = parts - .next() - .ok_or_else(|| SwitchyardError::Other("missing request path".into()))? - .to_string(); - - let raw_body = &bytes[body_start..body_end]; - let body = if raw_body.is_empty() { - Value::Null - } else { - serde_json::from_slice(raw_body) - .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))? - }; - Ok(HttpRequest { method, path, body }) -} - -fn request_shape(bytes: &[u8]) -> Option<(usize, usize)> { - let header_end = find_bytes(bytes, b"\r\n\r\n")?; - let headers = std::str::from_utf8(&bytes[..header_end]).ok()?; - let content_length = headers - .lines() - .filter_map(|line| line.split_once(':')) - .find_map(|(name, value)| { - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - Some((header_end, content_length)) -} - -fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn response_for_request(request: HttpRequest, counts: &mut MockCounts) -> Result { - counts.requests.push(request.clone()); - if request.method == "GET" && request.path.starts_with("/v1/endpoints/health") { - counts.health = counts.health.saturating_add(1); - return Ok(json!({ - "endpoint_health": { - "fast": {"status": "healthy", "last_latency_ms": 10.0}, - "slow": {"status": "degraded", "last_latency_ms": 1.0} - } - })); - } - - if request.method == "POST" && request.path == "/fast/v1/chat/completions" { - counts.fast = counts.fast.saturating_add(1); - counts.fast_body = Some(request.body.clone()); - return Ok(openai_completion("fast", request.body)); - } - - if request.method == "POST" && request.path == "/slow/v1/chat/completions" { - counts.slow = counts.slow.saturating_add(1); - counts.slow_body = Some(request.body.clone()); - return Ok(openai_completion("slow", request.body)); - } - - Err(SwitchyardError::Other(format!( - "unexpected mock request {} {}", - request.method, request.path - ))) -} - -fn openai_completion(endpoint: &'static str, request_body: Value) -> Value { - json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "model": request_body.get("model").cloned().unwrap_or(Value::Null), - "mock_endpoint": endpoint, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop" - }], - "usage": { - "prompt_tokens": 2, - "completion_tokens": 1, - "total_tokens": 3 - } - }) -} - -fn write_json_response(stream: &mut std::net::TcpStream, status: u16, body: Value) -> Result<()> { - let body = serde_json::to_string(&body) - .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?; - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}"))) -} - -#[test] -fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> { - let config = config(vec![ - target("fast", "upstream-fast")?, - target("slow", "upstream-slow")?, - ]); - - let profile = config.build()?; - - assert_eq!(profile.health_snapshot().len(), 2); - assert!(!profile.is_ready()); - Ok(()) -} - -#[tokio::test] -async fn profile_run_polls_and_routes_native_openai_call_to_healthy_target() -> Result<()> { - profile_stats_accumulator().reset()?; - let mut server = MockLatencyOpenAiServer::spawn(2)?; - let fast = openai_target( - "fast", - "upstream-fast-native", - &format!("{}/fast/v1", server.base_url()), - ); - let slow = openai_target( - "slow", - "upstream-slow-native", - &format!("{}/slow/v1", server.base_url()), - ); - let config = LatencyServiceProfileConfig { - latency_service_url: server.base_url(), - targets: vec![fast, slow], - poll_timeout_secs: 5.0, - max_retries: 0, - }; - let profile = config.build()?; - - let response = profile - .run(ProfileInput { - request: ChatRequest::openai_chat(json!({ - "model": "client-model", - "messages": [{"role": "user", "content": "route me"}], - "max_tokens": 8 - })), - metadata: RequestMetadata::default(), - }) - .await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("upstream-fast-native") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy")); - let response = response.response; - - match response { - ChatResponse::OpenAiCompletion(body) => { - assert_eq!(body.body()["mock_endpoint"], "fast"); - assert_eq!(body.body()["model"], "upstream-fast-native"); - } - _ => return Err(SwitchyardError::Other("unexpected response shape".into())), - } - let counts = server.finish()?; - assert_eq!(counts.health, 1); - assert_eq!(counts.fast, 1); - assert_eq!(counts.slow, 0); - assert_eq!(counts.requests.len(), 2); - assert_eq!(counts.requests[0].method, "GET"); - assert!(counts.requests[0].path.starts_with("/v1/endpoints/health?")); - assert!(counts.requests[0].path.contains("endpoint_ids=fast")); - assert!(counts.requests[0].path.contains("endpoint_ids=slow")); - assert_eq!(counts.requests[1].method, "POST"); - assert_eq!(counts.requests[1].path, "/fast/v1/chat/completions"); - assert!(!counts - .requests - .iter() - .any(|request| request.path == "/slow/v1/chat/completions")); - assert_eq!( - counts - .fast_body - .as_ref() - .and_then(|body| body.get("model")) - .and_then(Value::as_str), - Some("upstream-fast-native") - ); - - let stats = profile_stats_accumulator().snapshot()?; - let model = stats - .models - .get("upstream-fast-native") - .ok_or_else(|| SwitchyardError::Other("global v2 stats should include model".into()))?; - assert!(model.calls >= 1); - assert!(model.prompt_tokens >= 2); - assert!(model.completion_tokens >= 1); - Ok(()) -} - -#[test] -fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { - let config = config(vec![target("fast", "upstream-fast")?]); - - assert_eq!(LatencyServiceProfileConfig::PROFILE_TYPE, "latency-service"); - assert_eq!(config.profile_type(), "latency-service"); - - let old_poller_field = json!({ - "latency_service_url": "http://latency.test", - "targets": config.targets, - "poll_timeout_secs": 5.0, - "max_retries": 2, - "poll_interval_secs": 1.0, - }); - let error = serde_json::from_value::(old_poller_field) - .err() - .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?; - assert!(error.to_string().contains("unknown field")); - Ok(()) -} - -#[test] -fn invalid_profile_config_is_rejected_by_build() -> Result<()> { - let fast = target("fast", "upstream-fast")?; - let duplicate = config(vec![fast.clone(), fast]); - - match duplicate.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("duplicate target fast")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "duplicate targets should reject profile construction".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - - let mut bad_timeout = config(vec![target("fast", "upstream-fast")?]); - bad_timeout.poll_timeout_secs = f64::INFINITY; - match bad_timeout.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("poll_timeout_secs")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "invalid timeout should reject profile construction".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - Ok(()) -} diff --git a/crates/switchyard-components-v2/tests/llm_routing_profile.rs b/crates/switchyard-components-v2/tests/llm_routing_profile.rs deleted file mode 100644 index f14dcf73..00000000 --- a/crates/switchyard-components-v2/tests/llm_routing_profile.rs +++ /dev/null @@ -1,687 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Public API tests for the components-v2 LLM-routing profile config. - -use std::collections::BTreeMap; -use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener, TcpStream}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; - -use serde_json::{json, Value}; -use switchyard_components_v2::{ - profile_stats_accumulator, LlmRoutingProfileConfig, LlmRoutingTierMapping, Profile, - ProfileConfig, ProfileHooks, ProfileInput, RequestMetadata, -}; -use switchyard_core::{ - BackendFormat, ChatRequest, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError, -}; -use tokio::sync::Mutex as AsyncMutex; - -#[derive(Clone, Debug, PartialEq)] -struct ObservedRequest { - path: String, - headers: BTreeMap, - body: Value, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -enum BackendMode { - FixedStatus(u16), - WeakContextOverflowThenOk, -} - -struct MockOpenAiServer { - addr: SocketAddr, - requests: Arc>>, - shutdown: Arc, - handle: Option>, -} - -impl MockOpenAiServer { - fn spawn(classifier_arguments: Option, backend_status: u16) -> Result { - Self::spawn_with_backend_mode( - classifier_arguments, - BackendMode::FixedStatus(backend_status), - ) - } - - fn spawn_with_backend_mode( - classifier_arguments: Option, - backend_mode: BackendMode, - ) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?; - let addr = listener - .local_addr() - .map_err(|error| SwitchyardError::Other(format!("local_addr failed: {error}")))?; - let requests = Arc::new(Mutex::new(Vec::new())); - let shutdown = Arc::new(AtomicBool::new(false)); - let thread_requests = Arc::clone(&requests); - let thread_shutdown = Arc::clone(&shutdown); - let handle = thread::spawn(move || { - for stream in listener.incoming() { - if thread_shutdown.load(Ordering::SeqCst) { - break; - } - let Ok(mut stream) = stream else { - continue; - }; - let Ok(request) = read_request(&mut stream) else { - continue; - }; - if let Ok(mut requests) = thread_requests.lock() { - requests.push(request.clone()); - } - let response = response_for(&request, classifier_arguments.as_ref(), backend_mode); - let _ = write_json_response(&mut stream, response.0, response.1); - } - }); - Ok(Self { - addr, - requests, - shutdown, - handle: Some(handle), - }) - } - - fn base_url(&self) -> String { - format!("http://{}", self.addr) - } - - fn requests(&self) -> Result> { - self.requests - .lock() - .map(|requests| requests.clone()) - .map_err(|_| SwitchyardError::Other("requests mutex poisoned".to_string())) - } -} - -impl Drop for MockOpenAiServer { - fn drop(&mut self) { - self.shutdown.store(true, Ordering::SeqCst); - let _ = TcpStream::connect(self.addr); - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - } -} - -fn read_request(stream: &mut TcpStream) -> Result { - stream - .set_read_timeout(Some(Duration::from_secs(5))) - .map_err(|error| SwitchyardError::Other(format!("set timeout failed: {error}")))?; - let mut bytes = Vec::new(); - let mut buf = [0_u8; 1024]; - let header_end = loop { - let read = stream - .read(&mut buf) - .map_err(|error| SwitchyardError::Other(format!("read failed: {error}")))?; - if read == 0 { - return Err(SwitchyardError::Other( - "connection closed early".to_string(), - )); - } - bytes.extend_from_slice(&buf[..read]); - if let Some(header_end) = find_header_end(&bytes) { - break header_end; - } - }; - let headers = String::from_utf8_lossy(&bytes[..header_end]); - let mut lines = headers.lines(); - let request_line = lines - .next() - .ok_or_else(|| SwitchyardError::Other("missing request line".to_string()))?; - let path = request_line - .split_whitespace() - .nth(1) - .ok_or_else(|| SwitchyardError::Other("missing request path".to_string()))? - .to_string(); - let headers = headers - .lines() - .skip(1) - .filter_map(|line| { - let (name, value) = line.split_once(':')?; - Some((name.to_ascii_lowercase(), value.trim().to_string())) - }) - .collect::>(); - let content_length = headers - .get("content-length") - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - let body_start = header_end + 4; - while bytes.len().saturating_sub(body_start) < content_length { - let read = stream - .read(&mut buf) - .map_err(|error| SwitchyardError::Other(format!("body read failed: {error}")))?; - if read == 0 { - break; - } - bytes.extend_from_slice(&buf[..read]); - } - let body = if content_length == 0 { - Value::Null - } else { - serde_json::from_slice(&bytes[body_start..body_start + content_length]) - .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))? - }; - Ok(ObservedRequest { - path, - headers, - body, - }) -} - -fn find_header_end(bytes: &[u8]) -> Option { - bytes.windows(4).position(|window| window == b"\r\n\r\n") -} - -fn response_for( - request: &ObservedRequest, - classifier_arguments: Option<&Value>, - backend_mode: BackendMode, -) -> (u16, Value) { - if request.path.contains("/classifier/") { - return match classifier_arguments { - Some(arguments) => { - let tool_name = request.body["tool_choice"]["function"]["name"] - .as_str() - .unwrap_or("select_route"); - ( - 200, - json!({ - "id": "chatcmpl-llm-routing-classifier", - "object": "chat.completion", - "model": request.body["model"], - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "tool_calls": [{ - "id": "call_route", - "type": "function", - "function": { - "name": tool_name, - "arguments": arguments.to_string(), - }, - }], - }, - "finish_reason": "tool_calls", - }], - "usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18}, - }), - ) - } - None => ( - 200, - json!({ - "id": "chatcmpl-llm-routing-classifier", - "object": "chat.completion", - "model": request.body["model"], - "choices": [{"message": {"role": "assistant", "content": "not a tool call"}}], - }), - ), - }; - } - - if matches!(backend_mode, BackendMode::WeakContextOverflowThenOk) - && request.path.contains("/weak/") - { - return ( - 400, - json!({ - "error": { - "message": "prompt is too long", - "code": "context_length_exceeded", - "type": "invalid_request_error" - } - }), - ); - } - - if let BackendMode::FixedStatus(backend_status) = backend_mode { - if backend_status >= 400 { - return ( - backend_status, - json!({"error": {"message": "selected backend failed", "code": "backend_failed"}}), - ); - } - } - ( - 200, - json!({ - "id": "chatcmpl-llm-routing-backend", - "object": "chat.completion", - "model": request.body["model"], - "choices": [{"message": {"role": "assistant", "content": "ok"}}], - "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, - }), - ) -} - -fn write_json_response(stream: &mut TcpStream, status: u16, body: Value) -> Result<()> { - let body = serde_json::to_string(&body) - .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?; - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}"))) -} - -fn target(id: &str, model: &str, base_url: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - target.endpoint.base_url = Some(base_url.to_string()); - target.endpoint.api_key = Some("test-key".to_string()); - Ok(target) -} - -fn config(base_url: &str) -> Result { - Ok(LlmRoutingProfileConfig { - strong: target("strong", "frontier/model", &format!("{base_url}/strong/v1"))?, - weak: target("weak", "cheap/model", &format!("{base_url}/weak/v1"))?, - classifier: target( - "classifier", - "classifier/model", - &format!("{base_url}/classifier/v1"), - )?, - fallback_target_on_evict: LlmTargetId::new("strong")?, - profile_name: "coding_agent".to_string(), - classifier_min_confidence: 0.0, - classifier_fail_open: true, - classifier_recent_turn_window: 4, - classifier_max_tokens: 4096, - alignment_min_confidence: 0.85, - default_tier: None, - tier_mapping: None, - classifier_system_prompt: None, - classifier_tool_name: None, - classifier_tool_description: None, - classifier_tool_parameters: None, - }) -} - -fn request() -> ProfileInput { - ProfileInput { - request: ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [{"role": "user", "content": "list files"}], - })), - metadata: RequestMetadata::default(), - } -} - -fn medium_signals() -> Value { - json!({ - "recommended_tier": "medium", - "confidence": 0.9, - "abstain": false, - "turn_type": "exploration", - "code_modification_scope": "none", - "tool_call_count_estimate": 0, - "requires_codebase_context": false - }) -} - -fn complex_signals() -> Value { - json!({ - "recommended_tier": "complex", - "confidence": 0.9, - "abstain": false, - "turn_type": "debug", - "code_modification_scope": "cross_module", - "tool_call_count_estimate": 4, - "requires_codebase_context": true - }) -} - -fn stats_test_lock() -> &'static AsyncMutex<()> { - static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| AsyncMutex::new(())) -} - -#[test] -fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { - let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?; - let config = config(&server.base_url())?; - - assert_eq!(LlmRoutingProfileConfig::PROFILE_TYPE, "llm-routing"); - assert_eq!(config.profile_type(), "llm-routing"); - - let old_stats_toggle = json!({ - "strong": config.strong, - "weak": config.weak, - "classifier": config.classifier, - "stats": false, - }); - let error = serde_json::from_value::(old_stats_toggle) - .err() - .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?; - assert!(error.to_string().contains("unknown field")); - Ok(()) -} - -#[tokio::test] -async fn process_routes_medium_policy_to_weak_with_required_tool_call() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?; - let profile = config(&server.base_url())?.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!(processed.profile_input.request.model(), Some("cheap/model")); - assert_eq!(processed.decision.tier, "weak"); - assert_eq!(processed.decision.source, "policy_tier"); - let requests = server.requests()?; - assert_eq!(requests.len(), 1); - let classifier_body = &requests[0].body; - assert_eq!( - classifier_body["tools"][0]["function"]["name"], - "select_route" - ); - assert_eq!( - classifier_body["tool_choice"]["function"]["name"], - "select_route" - ); - assert_eq!(classifier_body["tools"][0]["function"]["strict"], true); - assert!(classifier_body.get("response_format").is_none()); - Ok(()) -} - -#[tokio::test] -async fn yaml_overrides_classifier_tool_contract_and_tier_mapping() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?; - let mut config = config(&server.base_url())?; - config.classifier_max_tokens = 64; - config.classifier_system_prompt = Some("Use the tool to pick a backend tier.".to_string()); - config.classifier_tool_name = Some("choose_model".to_string()); - config.classifier_tool_description = Some("Choose the target model tier.".to_string()); - config.classifier_tool_parameters = Some(json!({ - "type": "object", - "properties": {"recommended_tier": {"type": "string"}}, - "required": ["recommended_tier"], - "additionalProperties": false, - })); - config.tier_mapping = Some(LlmRoutingTierMapping { - simple: "weak".to_string(), - medium: "strong".to_string(), - complex: "strong".to_string(), - reasoning: "strong".to_string(), - }); - let profile = config.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - let requests = server.requests()?; - let classifier_body = &requests[0].body; - assert_eq!(classifier_body["max_tokens"], 64); - assert_eq!( - classifier_body["messages"][0]["content"], - "Use the tool to pick a backend tier." - ); - assert_eq!( - classifier_body["tools"][0]["function"]["name"], - "choose_model" - ); - assert_eq!( - classifier_body["tools"][0]["function"]["description"], - "Choose the target model tier." - ); - assert_eq!( - classifier_body["tool_choice"]["function"]["name"], - "choose_model" - ); - Ok(()) -} - -#[test] -fn config_rejects_non_strict_classifier_tool_schema() -> Result<()> { - let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?; - let mut config = config(&server.base_url())?; - config.classifier_tool_parameters = Some(json!({ - "type": "object", - "properties": {"recommended_tier": {"type": "string"}}, - "required": ["recommended_tier"], - "additionalProperties": true, - })); - - let error = config - .build() - .err() - .ok_or_else(|| SwitchyardError::Other("expected strict schema validation".to_string()))?; - - assert!(error.to_string().contains("additionalProperties")); - assert!(error.to_string().contains("strict tool calls")); - Ok(()) -} - -#[tokio::test] -async fn low_confidence_defaults_to_strong() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let mut signals = medium_signals(); - signals["recommended_tier"] = json!("simple"); - signals["confidence"] = json!(0.2); - let server = MockOpenAiServer::spawn(Some(signals), 200)?; - let mut config = config(&server.base_url())?; - config.classifier_min_confidence = 0.8; - let profile = config.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!(processed.decision.source, "low_confidence"); - Ok(()) -} - -#[tokio::test] -async fn classifier_failure_fails_open_with_visible_decision_source() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(None, 200)?; - let profile = config(&server.base_url())?.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!(processed.decision.source, "classifier_error_fall_open"); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_errors, 1); - Ok(()) -} - -#[tokio::test] -async fn classifier_failure_fails_closed_when_fail_open_is_disabled() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(None, 200)?; - let mut config = config(&server.base_url())?; - config.classifier_fail_open = false; - let profile = config.build()?; - - let error = profile - .process(request()) - .await - .err() - .ok_or_else(|| SwitchyardError::Other("expected classifier failure".to_string()))?; - - assert!(error.to_string().contains("LLM classifier failed")); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_errors, 1); - Ok(()) -} - -#[tokio::test] -async fn run_records_backend_and_classifier_stats() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(complex_signals()), 200)?; - let profile = config(&server.base_url())?.build()?; - - let response = profile.run(request()).await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong")); - assert_eq!(routing_metadata.confidence, Some(0.9)); - assert_eq!( - routing_metadata.router_version.as_deref(), - Some("llm-routing:coding_agent:v1") - ); - assert_eq!(routing_metadata.tolerance, Some(0.0)); - assert_eq!( - response.body().and_then(|body| body["model"].as_str()), - Some("frontier/model") - ); - let requests = server.requests()?; - assert_eq!(requests.len(), 2); - assert_eq!(requests[0].path, "/classifier/v1/chat/completions"); - assert_eq!(requests[1].path, "/strong/v1/chat/completions"); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_requests, 1); - assert!(stats.models.contains_key("frontier/model")); - assert!(stats.tiers.contains_key("strong")); - Ok(()) -} - -#[tokio::test] -async fn run_retries_fallback_target_after_context_window_overflow() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn_with_backend_mode( - Some(medium_signals()), - BackendMode::WeakContextOverflowThenOk, - )?; - let profile = config(&server.base_url())?.build()?; - - let response = profile.run(request()).await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong")); - assert!(routing_metadata - .rationale - .as_deref() - .is_some_and(|reason| reason.contains("context window"))); - assert_eq!( - response.body().and_then(|body| body["model"].as_str()), - Some("frontier/model") - ); - let requests = server.requests()?; - assert_eq!( - requests - .iter() - .map(|request| request.path.as_str()) - .collect::>(), - vec![ - "/classifier/v1/chat/completions", - "/weak/v1/chat/completions", - "/strong/v1/chat/completions", - ] - ); - assert_eq!(requests[1].body["model"], "cheap/model"); - assert_eq!(requests[2].body["model"], "frontier/model"); - Ok(()) -} - -#[tokio::test] -async fn run_normalizes_reasoning_effort_and_applies_provider_defaults() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?; - let mut config = config(&server.base_url())?; - config.classifier.model = ModelId::new("nvidia/deepseek-ai/deepseek-v4-flash")?; - config.weak.model = ModelId::new("nvidia/deepseek-ai/evals-deepseek-v4-pro")?; - let profile = config.build()?; - let mut input = request(); - input.request = ChatRequest::openai_chat(json!({ - "model": "client/model", - "messages": [{"role": "user", "content": "list files"}], - "reasoning_effort": "xhigh", - })); - - let response = profile.run(input).await?; - - assert_eq!( - response.body().and_then(|body| body["model"].as_str()), - Some("nvidia/deepseek-ai/evals-deepseek-v4-pro") - ); - let requests = server.requests()?; - let classifier = &requests[0]; - let backend = &requests[1]; - assert_eq!( - classifier.body["chat_template_kwargs"], - json!({"enable_thinking": false}) - ); - assert_eq!( - classifier.headers.get("x-inference-priority"), - Some(&"batch".to_string()) - ); - assert!(classifier.body["messages"][1]["content"] - .as_str() - .is_some_and(|content| content.contains("\"reasoning_effort\":\"high\""))); - assert_eq!(backend.body["reasoning_effort"], "high"); - assert_eq!( - backend.body["chat_template_kwargs"], - json!({"enable_thinking": false}) - ); - assert_eq!( - backend.headers.get("x-inference-priority"), - Some(&"batch".to_string()) - ); - Ok(()) -} - -#[tokio::test] -async fn run_records_selected_backend_failure() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(complex_signals()), 503)?; - let profile = config(&server.base_url())?.build()?; - - let error = - profile.run(request()).await.err().ok_or_else(|| { - SwitchyardError::Other("expected selected backend failure".to_string()) - })?; - - assert!(error.to_string().contains("selected backend failed")); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.total_errors, 1); - let model = stats - .models - .get("frontier/model") - .ok_or_else(|| SwitchyardError::Other("frontier/model stats missing".to_string()))?; - assert_eq!(model.errors, 1); - Ok(()) -} diff --git a/crates/switchyard-components-v2/tests/passthrough_profile.rs b/crates/switchyard-components-v2/tests/passthrough_profile.rs deleted file mode 100644 index 0e3cbded..00000000 --- a/crates/switchyard-components-v2/tests/passthrough_profile.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Public API tests for the components-v2 passthrough profile config. - -use serde_json::json; -use switchyard_components_v2::{PassthroughProfileConfig, ProfileConfig}; -use switchyard_core::{BackendFormat, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError}; - -fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) -} - -fn anthropic_target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::Anthropic; - Ok(target) -} - -fn auto_target(id: &str, model: &str) -> Result { - Ok(LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?)) -} - -fn config(target: LlmTarget) -> PassthroughProfileConfig { - PassthroughProfileConfig { target } -} - -#[test] -fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> { - let config = config(target("direct", "provider/model")?); - - let _profile = config.build()?; - Ok(()) -} - -#[test] -fn profile_config_build_supports_anthropic_native_backend_stack() -> Result<()> { - let config = config(anthropic_target("direct", "anthropic/model")?); - - let _profile = config.build()?; - Ok(()) -} - -#[test] -fn profile_config_build_rejects_unresolved_backend_format() -> Result<()> { - let config = config(auto_target("direct", "provider/model")?); - - match config.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("must have a resolved backend format")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "unresolved passthrough backend format should fail".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - Ok(()) -} - -#[test] -fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { - let config = config(target("direct", "provider/model")?); - - assert_eq!(PassthroughProfileConfig::PROFILE_TYPE, "passthrough"); - assert_eq!(config.profile_type(), "passthrough"); - - let old_stats_toggle = json!({ - "target": config.target, - "stats": false, - }); - let error = serde_json::from_value::(old_stats_toggle) - .err() - .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?; - assert!(error.to_string().contains("unknown field")); - Ok(()) -} diff --git a/crates/switchyard-components-v2/tests/random_routing_profile.rs b/crates/switchyard-components-v2/tests/random_routing_profile.rs deleted file mode 100644 index 66fd2cb5..00000000 --- a/crates/switchyard-components-v2/tests/random_routing_profile.rs +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Public API tests for the components-v2 random-routing profile config. - -use serde_json::json; -use switchyard_components_v2::{ProfileConfig, RandomRoutingProfileConfig}; -use switchyard_core::{BackendFormat, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError}; - -fn target(id: &str, model: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - Ok(target) -} - -fn config(strong: LlmTarget, weak: LlmTarget, probability: f64) -> RandomRoutingProfileConfig { - RandomRoutingProfileConfig { - strong, - weak, - strong_probability: probability, - rng_seed: Some(7), - } -} - -#[test] -fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> { - let config = config( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 0.5, - ); - - let _profile = config.build()?; - Ok(()) -} - -#[test] -fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { - let config = config( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 0.5, - ); - - assert_eq!(RandomRoutingProfileConfig::PROFILE_TYPE, "random-routing"); - assert_eq!(config.profile_type(), "random-routing"); - - let old_stats_toggle = json!({ - "strong": config.strong, - "weak": config.weak, - "strong_probability": 0.5, - "rng_seed": 7, - "stats": false, - }); - let error = serde_json::from_value::(old_stats_toggle) - .err() - .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?; - assert!(error.to_string().contains("unknown field")); - Ok(()) -} - -#[test] -fn invalid_probability_is_rejected_by_profile_config_build() -> Result<()> { - let config = config( - target("strong", "frontier/model")?, - target("weak", "cheap/model")?, - 1.5, - ); - - match config.build() { - Err(SwitchyardError::InvalidConfig(message)) => { - assert!(message.contains("strong_probability must be finite and in [0.0, 1.0]")); - } - Ok(_) => { - return Err(SwitchyardError::Other( - "invalid probability should reject profile construction".into(), - )); - } - Err(other) => { - return Err(SwitchyardError::Other(format!( - "expected InvalidConfig, got {other}" - ))); - } - } - Ok(()) -} diff --git a/crates/switchyard-components-v2/tests/stage_router_profile.rs b/crates/switchyard-components-v2/tests/stage_router_profile.rs deleted file mode 100644 index 7f57d34f..00000000 --- a/crates/switchyard-components-v2/tests/stage_router_profile.rs +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Public API tests for the components-v2 stage_router profile config. - -use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener, TcpStream}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::thread::{self, JoinHandle}; -use std::time::Duration; - -use serde_json::{json, Value}; -use switchyard_components_v2::{ - profile_stats_accumulator, Profile, ProfileConfig, ProfileHooks, ProfileInput, RequestMetadata, - StageRouterClassifierConfig, StageRouterDecisionSource, StageRouterPickerMode, - StageRouterProfileConfig, StageRouterTier, -}; -use switchyard_core::{ - BackendFormat, ChatRequest, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError, -}; -use tokio::sync::Mutex as AsyncMutex; - -#[derive(Clone, Debug, PartialEq)] -struct ObservedRequest { - path: String, - body: Value, -} - -enum MockResponse { - Json(u16, Value), - Raw(u16, &'static str), -} - -const NON_JSON_CLASSIFIER_RESPONSE: &str = "__switchyard_non_json_classifier_response__"; - -struct MockOpenAiServer { - addr: SocketAddr, - requests: Arc>>, - shutdown: Arc, - handle: Option>, -} - -impl MockOpenAiServer { - fn spawn(classifier_content: Option, backend_status: u16) -> Result { - let listener = TcpListener::bind("127.0.0.1:0") - .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?; - let addr = listener - .local_addr() - .map_err(|error| SwitchyardError::Other(format!("local_addr failed: {error}")))?; - let requests = Arc::new(Mutex::new(Vec::new())); - let shutdown = Arc::new(AtomicBool::new(false)); - let thread_requests = Arc::clone(&requests); - let thread_shutdown = Arc::clone(&shutdown); - let handle = thread::spawn(move || { - for stream in listener.incoming() { - if thread_shutdown.load(Ordering::SeqCst) { - break; - } - let Ok(mut stream) = stream else { - continue; - }; - let Ok(request) = read_request(&mut stream) else { - continue; - }; - if let Ok(mut requests) = thread_requests.lock() { - requests.push(request.clone()); - } - let response = - response_for(&request, classifier_content.as_deref(), backend_status); - let _ = write_response(&mut stream, response); - } - }); - Ok(Self { - addr, - requests, - shutdown, - handle: Some(handle), - }) - } - - fn base_url(&self) -> String { - format!("http://{}", self.addr) - } - - fn requests(&self) -> Result> { - self.requests - .lock() - .map(|requests| requests.clone()) - .map_err(|_| SwitchyardError::Other("requests mutex poisoned".to_string())) - } -} - -impl Drop for MockOpenAiServer { - fn drop(&mut self) { - self.shutdown.store(true, Ordering::SeqCst); - let _ = TcpStream::connect(self.addr); - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - } -} - -fn read_request(stream: &mut TcpStream) -> Result { - stream - .set_read_timeout(Some(Duration::from_secs(5))) - .map_err(|error| SwitchyardError::Other(format!("set timeout failed: {error}")))?; - let mut bytes = Vec::new(); - let mut buf = [0_u8; 1024]; - let header_end = loop { - let read = stream - .read(&mut buf) - .map_err(|error| SwitchyardError::Other(format!("read failed: {error}")))?; - if read == 0 { - return Err(SwitchyardError::Other( - "connection closed early".to_string(), - )); - } - bytes.extend_from_slice(&buf[..read]); - if let Some(header_end) = find_header_end(&bytes) { - break header_end; - } - }; - let headers = String::from_utf8_lossy(&bytes[..header_end]); - let mut lines = headers.lines(); - let request_line = lines - .next() - .ok_or_else(|| SwitchyardError::Other("missing request line".to_string()))?; - let path = request_line - .split_whitespace() - .nth(1) - .ok_or_else(|| SwitchyardError::Other("missing request path".to_string()))? - .to_string(); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - let body_start = header_end + 4; - while bytes.len().saturating_sub(body_start) < content_length { - let read = stream - .read(&mut buf) - .map_err(|error| SwitchyardError::Other(format!("body read failed: {error}")))?; - if read == 0 { - break; - } - bytes.extend_from_slice(&buf[..read]); - } - let body_end = body_start + content_length; - if bytes.len() < body_end { - return Err(SwitchyardError::Other( - "connection closed before full body was read".to_string(), - )); - } - let body = if content_length == 0 { - Value::Null - } else { - serde_json::from_slice(&bytes[body_start..body_end]) - .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))? - }; - Ok(ObservedRequest { path, body }) -} - -fn find_header_end(bytes: &[u8]) -> Option { - bytes.windows(4).position(|window| window == b"\r\n\r\n") -} - -fn response_for( - request: &ObservedRequest, - classifier_content: Option<&str>, - backend_status: u16, -) -> MockResponse { - if request.path.contains("/classifier/") { - if classifier_content == Some(NON_JSON_CLASSIFIER_RESPONSE) { - return MockResponse::Raw(200, "not json"); - } - return MockResponse::Json( - 200, - json!({ - "id": "chatcmpl-stage_router-classifier", - "object": "chat.completion", - "model": request.body["model"], - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": classifier_content.unwrap_or("not json"), - }, - "finish_reason": "stop", - }], - "usage": {"prompt_tokens": 11, "completion_tokens": 1, "total_tokens": 12}, - }), - ); - } - - if backend_status >= 400 { - return MockResponse::Json( - backend_status, - json!({"error": {"message": "selected backend failed", "code": "backend_failed"}}), - ); - } - MockResponse::Json( - 200, - json!({ - "id": "chatcmpl-stage_router-backend", - "object": "chat.completion", - "model": request.body["model"], - "choices": [{"message": {"role": "assistant", "content": "ok"}}], - "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, - }), - ) -} - -fn write_response(stream: &mut TcpStream, response: MockResponse) -> Result<()> { - match response { - MockResponse::Json(status, body) => write_json_response(stream, status, body), - MockResponse::Raw(status, body) => write_raw_response(stream, status, body), - } -} - -fn write_json_response(stream: &mut TcpStream, status: u16, body: Value) -> Result<()> { - let body = serde_json::to_string(&body) - .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?; - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}"))) -} - -fn write_raw_response(stream: &mut TcpStream, status: u16, body: &'static str) -> Result<()> { - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: text/plain\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(response.as_bytes()) - .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}"))) -} - -fn target(id: &str, model: &str, base_url: &str) -> Result { - let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); - target.format = BackendFormat::OpenAi; - target.endpoint.base_url = Some(base_url.to_string()); - target.endpoint.api_key = Some("test-key".to_string()); - Ok(target) -} - -fn config(base_url: &str) -> Result { - Ok(StageRouterProfileConfig { - capable: target( - "capable", - "frontier/model", - &format!("{base_url}/capable/v1"), - )?, - efficient: target( - "efficient", - "cheap/model", - &format!("{base_url}/efficient/v1"), - )?, - fallback_target_on_evict: LlmTargetId::new("capable")?, - picker: StageRouterPickerMode::CapableFirst, - confidence_threshold: 0.7, - signal_recent_window: 3, - classifier: Some(StageRouterClassifierConfig { - model: "classifier/model".to_string(), - api_key: "test-key".to_string(), - base_url: Some(format!("{base_url}/classifier/v1")), - timeout_secs: 1.0, - recent_turn_window: 3, - max_tokens: 4096, - system_prompt: None, - }), - enable_stats: true, - }) -} - -fn profile_input(request: ChatRequest) -> ProfileInput { - ProfileInput { - request, - metadata: RequestMetadata::default(), - } -} - -fn request() -> ProfileInput { - profile_input(ChatRequest::openai_chat(json!({ - "model": "client/stage_router", - "messages": [{"role": "user", "content": "hello"}], - }))) -} - -fn request_with_tool_result(tool_name: &str, content: &str) -> ProfileInput { - profile_input(ChatRequest::openai_chat(json!({ - "model": "client/stage_router", - "messages": [ - {"role": "user", "content": "work on this"}, - {"role": "assistant", "tool_calls": [{"function": {"name": tool_name}}]}, - {"role": "tool", "tool_call_id": "1", "content": content}, - ], - }))) -} - -fn tier_content(tier: &str) -> String { - json!({"tier": tier}).to_string() -} - -fn stats_test_lock() -> &'static AsyncMutex<()> { - static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| AsyncMutex::new(())) -} - -#[tokio::test] -async fn critical_severity_overrides_to_capable_without_classifier() -> Result<()> { - let server = MockOpenAiServer::spawn(None, 200)?; - let mut config = config(&server.base_url())?; - config.classifier = None; - let profile = config.build()?; - - let processed = profile - .process(request_with_tool_result("Bash", "out of memory")) - .await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!(processed.decision.tier, StageRouterTier::Capable); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::Override - ); - assert!(server.requests()?.is_empty()); - Ok(()) -} - -#[tokio::test] -async fn negative_score_routes_to_efficient_without_classifier() -> Result<()> { - let server = MockOpenAiServer::spawn(None, 200)?; - let mut config = config(&server.base_url())?; - config.classifier = None; - config.confidence_threshold = 0.1; - let profile = config.build()?; - - let processed = profile - .process(request_with_tool_result("Write", "ok")) - .await?; - - assert_eq!(processed.profile_input.request.model(), Some("cheap/model")); - assert_eq!(processed.decision.tier, StageRouterTier::Efficient); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::Dimensions - ); - Ok(()) -} - -#[tokio::test] -async fn low_confidence_uses_classifier_when_configured() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(tier_content("efficient")), 200)?; - let profile = config(&server.base_url())?.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!(processed.profile_input.request.model(), Some("cheap/model")); - assert_eq!(processed.decision.tier, StageRouterTier::Efficient); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::LlmClassifier - ); - let requests = server.requests()?; - assert_eq!(requests.len(), 1); - let classifier_body = &requests[0].body; - assert_eq!(classifier_body["max_tokens"], 4096); - assert!(classifier_body["messages"][0]["content"] - .as_str() - .is_some_and( - |content| content.contains("routing classifier inside an agentic coding stage-router") - )); - assert_eq!(classifier_body["response_format"]["type"], "json_object"); - Ok(()) -} - -#[tokio::test] -async fn yaml_overrides_classifier_prompt_and_max_tokens() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(tier_content("efficient")), 200)?; - let mut config = config(&server.base_url())?; - let classifier = config - .classifier - .as_mut() - .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))?; - classifier.max_tokens = 64; - classifier.system_prompt = Some("Pick a stage_router tier.".to_string()); - let profile = config.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!(processed.profile_input.request.model(), Some("cheap/model")); - let requests = server.requests()?; - let classifier_body = &requests[0].body; - assert_eq!(classifier_body["max_tokens"], 64); - assert_eq!( - classifier_body["messages"][0]["content"], - "Pick a stage_router tier." - ); - Ok(()) -} - -#[test] -fn config_rejects_bad_classifier_options() -> Result<()> { - let mut zero_tokens = config("http://127.0.0.1:9")?; - zero_tokens - .classifier - .as_mut() - .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))? - .max_tokens = 0; - let error = zero_tokens - .build() - .err() - .ok_or_else(|| SwitchyardError::Other("expected max token validation failure".into()))?; - assert!(error.to_string().contains("classifier.max_tokens")); - - let mut empty_prompt = config("http://127.0.0.1:9")?; - empty_prompt - .classifier - .as_mut() - .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))? - .system_prompt = Some(" ".to_string()); - let error = empty_prompt - .build() - .err() - .ok_or_else(|| SwitchyardError::Other("expected prompt validation failure".into()))?; - assert!(error.to_string().contains("classifier.system_prompt")); - Ok(()) -} - -#[tokio::test] -async fn malformed_classifier_falls_open_to_default() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(None, 200)?; - let profile = config(&server.base_url())?.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::FallOpen - ); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_errors, 1); - Ok(()) -} - -#[tokio::test] -async fn non_json_classifier_falls_open_and_records_error() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(NON_JSON_CLASSIFIER_RESPONSE.to_string()), 200)?; - let profile = config(&server.base_url())?.build()?; - - let processed = profile.process(request()).await?; - - assert_eq!( - processed.profile_input.request.model(), - Some("frontier/model") - ); - assert_eq!( - processed.decision.source, - StageRouterDecisionSource::FallOpen - ); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_errors, 1); - Ok(()) -} - -#[tokio::test] -async fn run_records_backend_and_classifier_stats() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(tier_content("capable")), 200)?; - let profile = config(&server.base_url())?.build()?; - - let response = profile.run(request()).await?; - - let routing_metadata = response - .routing_metadata - .as_ref() - .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?; - assert_eq!( - routing_metadata.selected_model.as_deref(), - Some("frontier/model") - ); - assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable")); - assert_eq!(routing_metadata.confidence, None); - assert!(routing_metadata - .rationale - .as_deref() - .is_some_and(|reason| reason.contains("source=llm-classifier"))); - assert_eq!( - response.body().and_then(|body| body["model"].as_str()), - Some("frontier/model") - ); - let requests = server.requests()?; - assert_eq!(requests.len(), 2); - assert_eq!(requests[0].path, "/classifier/v1/chat/completions"); - assert_eq!(requests[1].path, "/capable/v1/chat/completions"); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.classifier.total_requests, 1); - assert!(stats.models.contains_key("frontier/model")); - assert!(stats.tiers.contains_key("capable")); - Ok(()) -} - -#[tokio::test] -async fn run_records_selected_backend_failure() -> Result<()> { - let _guard = stats_test_lock().lock().await; - profile_stats_accumulator().reset()?; - let server = MockOpenAiServer::spawn(Some(tier_content("capable")), 503)?; - let profile = config(&server.base_url())?.build()?; - - let error = - profile.run(request()).await.err().ok_or_else(|| { - SwitchyardError::Other("expected selected backend failure".to_string()) - })?; - - assert!(error.to_string().contains("selected backend failed")); - let stats = profile_stats_accumulator().snapshot()?; - assert_eq!(stats.total_errors, 1); - let model = stats - .models - .get("frontier/model") - .ok_or_else(|| SwitchyardError::Other("frontier/model stats missing".to_string()))?; - assert_eq!(model.errors, 1); - Ok(()) -} diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs index 5bd2e718..3b34c631 100644 --- a/crates/switchyard-components/src/lib.rs +++ b/crates/switchyard-components/src/lib.rs @@ -5,7 +5,7 @@ //! //! `switchyard-core` owns traits and wire wrappers. This crate owns built-in //! compatibility implementations: backends, request processors, and response -//! processors. New Rust orchestration belongs in components-v2 profiles. +//! processors. New Rust orchestration belongs in libsy algorithms and clients. pub mod backends; pub mod dimension_collector; diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml index fbd9fba9..0c6fa707 100644 --- a/crates/switchyard-py/Cargo.toml +++ b/crates/switchyard-py/Cargo.toml @@ -27,7 +27,6 @@ serde_json = "1" switchyard-core = { path = "../switchyard-core" } switchyard-protocol = { path = "../protocol" } switchyard-components = { path = "../switchyard-components" } -switchyard-components-v2 = { path = "../switchyard-components-v2" } switchyard-translation = { path = "../switchyard-translation" } tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync"] } tracing = { version = "0.1", default-features = false, features = ["std"] } diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 5ef43c27..266aa858 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -7,7 +7,6 @@ mod component_bindings; mod core_bindings; mod errors; mod libsy_bindings; -mod profile_bindings; mod py_serde; mod translation; @@ -18,6 +17,5 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> { core_bindings::register(module)?; component_bindings::register(module)?; libsy_bindings::register(module)?; - profile_bindings::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/profile_bindings/mod.rs b/crates/switchyard-py/src/profile_bindings/mod.rs deleted file mode 100644 index eb082284..00000000 --- a/crates/switchyard-py/src/profile_bindings/mod.rs +++ /dev/null @@ -1,334 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for the components-v2 profile API. -//! -//! Two surfaces are exposed: -//! -//! - The erased serving surface — [`PyProfileConfigDocument`], [`PyProfileConfigPlan`], -//! and [`PyProfile`] — loads config files and runs any profile through the -//! object-safe `Profile` contract. It is the right default for servers, -//! launchers, and generic profile tables that only need `run()`. -//! - The typed request-input surface (see [`typed`]) exposes `ProfileInput` and -//! `ProfileRequestMetadata` so Python-authored profiles can share the same -//! request envelope as Rust-authored profiles. -//! -//! Concrete profile authoring now belongs to `switchyard.lib.profiles` on the -//! Python side or `switchyard-components-v2` on the Rust side; this module only -//! bridges config-file execution and the shared input value. - -mod typed; - -use std::path::PathBuf; -use std::sync::Arc; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; -use switchyard_components_v2::{ - parse_profile_config_path as parse_v2_profile_config_path, - parse_profile_config_str as parse_v2_profile_config_str, Profile, ProfileConfigDocument, - ProfileConfigFormat, ProfileConfigPlan, ProfileInput, -}; -use switchyard_core::{LlmTargetId, ProfileId, SwitchyardError}; - -use crate::component_bindings::config::PyLlmTarget; -use crate::core_bindings::request::PyChatRequest; -use crate::core_bindings::response::PyChatResponse; -use crate::errors::py_core_error; -use crate::py_serde::value_to_python; - -use self::typed::PyProfileRequestMetadata; - -/// Parsed components-v2 profile config document. -#[pyclass(name = "ProfileConfigDocument", frozen, skip_from_py_object)] -#[derive(Clone, Debug)] -struct PyProfileConfigDocument { - inner: ProfileConfigDocument, -} - -#[pymethods] -impl PyProfileConfigDocument { - /// Return parsed profile IDs in deterministic order. - fn profile_ids(&self) -> Vec { - self.inner - .profile_ids() - .map(|profile_id| profile_id.as_str().to_string()) - .collect() - } - - /// Return a parsed profile's type discriminator, or `None`. - fn profile_type(&self, profile_id: &str) -> PyResult> { - let profile_id = parse_profile_id(profile_id)?; - Ok(self - .inner - .profile_type(&profile_id) - .map(std::borrow::ToOwned::to_owned)) - } - - /// Return a parsed profile's envelope `subagent_target` reference, or `None`. - fn profile_subagent_target(&self, profile_id: &str) -> PyResult> { - let profile_id = parse_profile_id(profile_id)?; - Ok(self - .inner - .profile_subagent_target(&profile_id) - .map(|target_id| target_id.as_str().to_owned())) - } - - /// Return a parsed profile's body without the `type` discriminator. - fn profile_body(&self, py: Python<'_>, profile_id: &str) -> PyResult>> { - let profile_id = parse_profile_id(profile_id)?; - self.inner - .profile_body(&profile_id) - .map(|body| value_to_python(py, body)) - .transpose() - } - - /// Return a copy with selected profiles removed before resolution. - fn without_profiles(&self, profile_ids: Vec) -> PyResult { - let profile_ids = profile_ids - .iter() - .map(|profile_id| parse_profile_id(profile_id)) - .collect::>>()?; - Ok(Self { - inner: self.inner.without_profiles(&profile_ids), - }) - } - - /// Resolve endpoint/target references and validate profile-owned configs. - fn resolve(&self) -> PyResult { - self.inner - .resolve() - .map(PyProfileConfigPlan::from_core) - .map_err(py_core_error) - } - - fn __repr__(&self) -> &'static str { - "ProfileConfigDocument()" - } -} - -/// Resolved components-v2 profile config plan. -#[pyclass(name = "ProfileConfigPlan", frozen, skip_from_py_object)] -#[derive(Clone)] -struct PyProfileConfigPlan { - inner: ProfileConfigPlan, -} - -impl PyProfileConfigPlan { - fn from_core(inner: ProfileConfigPlan) -> Self { - Self { inner } - } -} - -#[pymethods] -impl PyProfileConfigPlan { - /// Return resolved profile IDs in deterministic order. - fn profile_ids(&self) -> Vec { - self.inner - .profile_ids() - .map(|profile_id| profile_id.as_str().to_string()) - .collect() - } - - /// Return resolved target IDs in deterministic order. - fn target_ids(&self) -> Vec { - self.inner - .targets() - .map(|(target_id, _target)| target_id.as_str().to_string()) - .collect() - } - - /// Return the serialized profile type for one profile ID, or `None`. - fn profile_type(&self, profile_id: &str) -> PyResult> { - let profile_id = parse_profile_id(profile_id)?; - Ok(self - .inner - .profile_type(&profile_id) - .map(std::borrow::ToOwned::to_owned)) - } - - /// Return a resolved target by ID, or `None`. - fn target(&self, target_id: &str) -> PyResult> { - let target_id = parse_target_id(target_id)?; - Ok(self - .inner - .target(&target_id) - .cloned() - .map(PyLlmTarget::from_core)) - } - - /// Build one profile runtime by profile ID. - fn build_profile(&self, profile_id: &str) -> PyResult { - let profile_id = parse_profile_id(profile_id)?; - let profile = self - .inner - .build_profile(&profile_id) - .map_err(py_core_error)?; - Ok(PyProfile::from_boxed( - profile_id.as_str().to_string(), - profile, - )) - } - - /// Build all profile runtimes in deterministic profile-ID order. - fn build_profiles<'py>(&self, py: Python<'py>) -> PyResult> { - let profiles = self.inner.build_profiles().map_err(py_core_error)?; - let out = PyDict::new(py); - for (profile_id, profile) in profiles { - out.set_item( - profile_id.as_str(), - Py::new( - py, - PyProfile::from_boxed(profile_id.as_str().to_string(), profile), - )?, - )?; - } - Ok(out) - } - - fn __repr__(&self) -> String { - format!( - "ProfileConfigPlan(profiles={}, targets={})", - self.inner.profile_count(), - self.inner.target_count(), - ) - } -} - -/// Object-safe components-v2 profile serving runtime. -#[pyclass(name = "Profile", frozen, skip_from_py_object)] -#[derive(Clone)] -struct PyProfile { - profile_id: String, - inner: Arc, -} - -impl PyProfile { - fn from_boxed(profile_id: String, profile: Box) -> Self { - Self { - profile_id, - inner: Arc::from(profile), - } - } -} - -#[pymethods] -impl PyProfile { - /// Profile ID this runtime was built from. - #[getter] - fn profile_id(&self) -> String { - self.profile_id.clone() - } - - /// Execute the full profile-owned request lifecycle. - #[pyo3(signature = (request, metadata=None))] - fn run<'py>( - &self, - py: Python<'py>, - request: PyRef<'_, PyChatRequest>, - metadata: Option>, - ) -> PyResult> { - let profile = Arc::clone(&self.inner); - let request = request.clone_core(); - let metadata = metadata - .map(|metadata| metadata.clone_core()) - .unwrap_or_default(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - // The erased profile is the metadata-free generic entry point used - // by config plans and servers. Python-authored profiles can still - // carry metadata through the shared `ProfileInput` binding. - let profile_response = profile - .run(ProfileInput { request, metadata }) - .await - .map_err(py_core_error)?; - let (response, _routing_metadata) = profile_response.into_parts(); - Python::attach(|py| { - Py::new(py, PyChatResponse::from_core(py, response)?) - .map(|response| response.into_any()) - }) - }) - } - - fn __repr__(&self) -> String { - format!("Profile(profile_id={:?})", self.profile_id) - } -} - -/// Parse a components-v2 profile config string. -#[pyfunction(name = "parse_profile_config_str")] -#[pyo3(signature = (input, format = "yaml"))] -fn py_parse_profile_config_str(input: &str, format: &str) -> PyResult { - let format = profile_config_format_from_str(format)?; - parse_v2_profile_config_str(input, format) - .map(|inner| PyProfileConfigDocument { inner }) - .map_err(py_core_error) -} - -/// Parse a components-v2 profile config file, inferring the format from extension. -#[pyfunction(name = "parse_profile_config_path")] -fn py_parse_profile_config_path(path: PathBuf) -> PyResult { - parse_v2_profile_config_path(path) - .map(|inner| PyProfileConfigDocument { inner }) - .map_err(py_core_error) -} - -/// Report whether normalized request headers mark delegated sub-agent work. -/// -/// A thin binding over the canonical detection and routing policy owned by -/// the protocol crate: `switchyard_protocol::Metadata::from_headers` for the -/// lineage fact (explicit `x-switchyard-is-subagent`, Claude Code agent -/// lineage, Codex/relay markers) and `Metadata::is_subagent_work` for the -/// work-vs-maintenance kind policy, so Codex `compact` / -/// `memory_consolidation` turns stay on normal profile routing. -#[pyfunction] -fn is_subagent_request(headers: &Bound<'_, PyAny>) -> PyResult { - let headers = typed::metadata_headers_from_python(headers)?; - // Lineage headers are single-valued; keep the first value when repeated. - let headers: std::collections::BTreeMap = headers - .into_iter() - .filter_map(|(name, values)| values.into_iter().next().map(|value| (name, value))) - .collect(); - Ok(switchyard_protocol::Metadata::from_headers(&headers).is_subagent_work()) -} - -/// Parse and resolve a components-v2 profile config file. -#[pyfunction] -fn load_profile_config(path: PathBuf) -> PyResult { - parse_v2_profile_config_path(path) - .and_then(|document| document.resolve()) - .map(PyProfileConfigPlan::from_core) - .map_err(py_core_error) -} - -fn profile_config_format_from_str(raw: &str) -> PyResult { - match raw { - "json" => Ok(ProfileConfigFormat::Json), - "toml" => Ok(ProfileConfigFormat::Toml), - "yaml" | "yml" => Ok(ProfileConfigFormat::Yaml), - other => Err(PyValueError::new_err(format!( - "unknown profile config format {other:?}; expected 'yaml', 'json', or 'toml'" - ))), - } -} - -fn parse_profile_id(value: &str) -> PyResult { - ProfileId::new(value).map_err(|error| py_core_error(SwitchyardError::from(error))) -} - -fn parse_target_id(value: &str) -> PyResult { - LlmTargetId::new(value).map_err(|error| py_core_error(SwitchyardError::from(error))) -} - -/// Registers profile bindings with the native Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(py_parse_profile_config_str, module)?)?; - module.add_function(wrap_pyfunction!(py_parse_profile_config_path, module)?)?; - module.add_function(wrap_pyfunction!(load_profile_config, module)?)?; - module.add_function(wrap_pyfunction!(is_subagent_request, module)?)?; - typed::register(module)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/profile_bindings/typed.rs b/crates/switchyard-py/src/profile_bindings/typed.rs deleted file mode 100644 index 2a905bee..00000000 --- a/crates/switchyard-py/src/profile_bindings/typed.rs +++ /dev/null @@ -1,223 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Typed request input bindings shared by Rust and Python profile runtimes. -//! -//! Concrete profile config/runtime bindings intentionally live outside this -//! module now. Python owns its profile authoring API, while the profile-config -//! loader exposes Rust-defined profiles through the erased `Profile` runtime in -//! `mod.rs`. - -use std::collections::BTreeMap; - -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyType}; -use switchyard_components_v2::{ProfileInput, RequestMetadata}; -use switchyard_core::RequestId; - -use crate::core_bindings::request::{request_type_from_python, request_type_object, PyChatRequest}; -use crate::py_serde::value_to_python; - -/// Endpoint-style metadata for direct Python profile calls. -#[pyclass(name = "ProfileRequestMetadata", frozen, skip_from_py_object)] -#[derive(Clone, Default)] -pub(crate) struct PyProfileRequestMetadata { - inner: RequestMetadata, -} - -impl PyProfileRequestMetadata { - fn from_core(inner: RequestMetadata) -> Self { - Self { inner } - } - - pub(crate) fn clone_core(&self) -> RequestMetadata { - self.inner.clone() - } -} - -#[pymethods] -impl PyProfileRequestMetadata { - #[new] - #[pyo3(signature = (request_id=None, inbound_format=None, headers=None))] - fn new( - request_id: Option, - inbound_format: Option<&Bound<'_, PyAny>>, - headers: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - Ok(Self { - inner: RequestMetadata { - request_id: parse_request_id(request_id)?, - inbound_format: inbound_format - .filter(|value| !value.is_none()) - .map(request_type_from_python) - .transpose()?, - headers: match headers { - Some(headers) if !headers.is_none() => metadata_headers_from_python(headers)?, - _ => BTreeMap::new(), - }, - }, - }) - } - - /// Build metadata from headers, inferring `request_id` from `x-request-id`. - #[classmethod] - #[pyo3(signature = (headers, inbound_format=None))] - fn from_headers( - _cls: &Bound<'_, PyType>, - headers: &Bound<'_, PyAny>, - inbound_format: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - let headers = metadata_headers_from_python(headers)?; - let request_id = headers - .get("x-request-id") - .and_then(|values| values.first()) - .cloned(); - Ok(Self { - inner: RequestMetadata { - request_id: parse_request_id(request_id)?, - inbound_format: inbound_format - .filter(|value| !value.is_none()) - .map(request_type_from_python) - .transpose()?, - headers, - }, - }) - } - - #[getter] - fn request_id(&self) -> Option { - self.inner - .request_id - .as_ref() - .map(|request_id| request_id.as_str().to_string()) - } - - #[getter] - fn inbound_format(&self, py: Python<'_>) -> PyResult>> { - self.inner - .inbound_format - .map(|request_type| request_type_object(py, request_type)) - .transpose() - } - - #[getter] - fn headers(&self, py: Python<'_>) -> PyResult> { - value_to_python( - py, - &serde_json::to_value(&self.inner.headers) - .map_err(|error| PyValueError::new_err(error.to_string()))?, - ) - } - - /// Convert metadata into a plain Python dictionary. - fn to_dict(&self, py: Python<'_>) -> PyResult> { - let dict = PyDict::new(py); - dict.set_item("request_id", self.request_id())?; - dict.set_item( - "inbound_format", - self.inner - .inbound_format - .map(crate::core_bindings::request::request_type_name), - )?; - dict.set_item("headers", self.headers(py)?)?; - Ok(dict.unbind().into_any()) - } - - fn __repr__(&self) -> String { - format!("ProfileRequestMetadata(request_id={:?})", self.request_id()) - } -} - -/// Rust-owned request input for direct Python profile calls. -#[pyclass(name = "ProfileInput", frozen, skip_from_py_object)] -#[derive(Clone)] -pub(crate) struct PyProfileInput { - inner: ProfileInput, -} - -#[pymethods] -impl PyProfileInput { - #[new] - #[pyo3(signature = (request, metadata=None))] - fn new( - request: PyRef<'_, PyChatRequest>, - metadata: Option>, - ) -> Self { - Self { - inner: ProfileInput { - request: request.clone_core(), - metadata: metadata_to_core(metadata), - }, - } - } - - /// Provider-neutral chat request carried by this profile input. - #[getter] - fn request(&self) -> PyChatRequest { - PyChatRequest::from_core(self.inner.request.clone()) - } - - /// Endpoint-style metadata carried by this profile input. - #[getter] - fn metadata(&self) -> PyProfileRequestMetadata { - PyProfileRequestMetadata::from_core(self.inner.metadata.clone()) - } - - fn __repr__(&self) -> String { - format!( - "ProfileInput(request_model={:?}, request_id={:?})", - self.inner.request.model(), - self.inner - .metadata - .request_id - .as_ref() - .map(|request_id| request_id.as_str()) - ) - } -} - -fn metadata_to_core(metadata: Option>) -> RequestMetadata { - metadata - .map(|metadata| metadata.clone_core()) - .unwrap_or_default() -} - -fn parse_request_id(request_id: Option) -> PyResult> { - request_id - .map(RequestId::new) - .transpose() - .map_err(|error| PyValueError::new_err(format!("invalid request_id: {error}"))) -} - -pub(crate) fn metadata_headers_from_python( - headers: &Bound<'_, PyAny>, -) -> PyResult>> { - let dict = PyDict::new(headers.py()); - dict.call_method1("update", (headers,))?; - let mut out = BTreeMap::>::new(); - for (key, value) in dict.iter() { - let key = key.extract::()?.to_ascii_lowercase(); - let values = metadata_header_values_from_python(&value)?; - out.entry(key).or_default().extend(values); - } - Ok(out) -} - -fn metadata_header_values_from_python(value: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(value) = value.extract::() { - return Ok(vec![value]); - } - value.extract::>().map_err(|_| { - PyTypeError::new_err( - "ProfileRequestMetadata headers must map strings to strings or lists of strings", - ) - }) -} - -/// Registers typed request input bindings with the native Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/docs/cli_reference.md b/docs/cli_reference.md index e02fc8d1..864225a0 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -6,7 +6,7 @@ This page is the canonical reference for every `switchyard` subcommand. It mirro | Verb | Audience | What it does | |---|---|---| -| [`serve`](#switchyard-serve) | Ops | Long-running proxy server. Serve a v2 profile config with `serve --config`; each profile id and target id appears on `GET /v1/models`, and clients select one by setting the request `model`. | +| [`serve`](#switchyard-serve) | Ops | Long-running proxy server. Serve a YAML route bundle selected with the global `--routing-profiles` flag. | | [`launch claude`](#switchyard-launch-claude) | Dev | Spawns Claude Code against a local proxy. Auto-picks a free port, sets `ANTHROPIC_*` env vars, tears the proxy down when Claude exits. `--smoke` runs a one-shot harness round-trip and exits. | | [`launch codex`](#switchyard-launch-codex) | Dev | Spawns OpenAI Codex CLI against a local proxy and injects a transient `switchyard` provider via repeated `-c` flags. `--smoke` runs a one-shot Codex round-trip. | | [`launch openclaw`](#switchyard-launch-openclaw) | Dev | Spawns `openclaw chat` against a local proxy using a transient `openclaw.json`; the user's OpenClaw configuration is untouched. `--smoke` runs a one-shot agent turn. | @@ -20,8 +20,8 @@ These apply to the top-level `switchyard` command, before any verb. | Flag | Purpose | |---|---| | `--version` | Print the installed Switchyard version (`switchyard X.Y.Z`) and exit. Reads the version from the installed package metadata. | -| `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. | -| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config`. | +| `--routing-profiles PATH` / `-c PATH` | [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. | +| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. | | `--rl-log-dir DIR` | Output directory for `--enable-rl-logging` traces (default: `./rl_data`). No effect without `--enable-rl-logging`. | ## Cross-cutting flag families @@ -60,22 +60,20 @@ startup does not require capability probes. ### Routing -Legacy routing policies that used to be standalone CLI verbs live in -routing-profile YAML files. Route bundles are deprecated; use v2 profile -configs for new `serve` setups. Two flags drive routing on `serve` and the -launchers: +Routing policies live in routing-profile YAML files. Two flags drive routing +on the launchers; `serve` uses `--routing-profiles`: | Flag | Purpose | |---|---| | `--model ID` | Single-model passthrough. Every request is rewritten to `model=ID` and forwarded to `--base-url`. | -| `--routing-profiles PATH` | Deprecated path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `passthrough`, `random_routing`, `stage_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. | +| `--routing-profiles PATH` | Path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `passthrough`, `random_routing`, `stage_router`, `escalation_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. | On the launchers, the two flags are mutually exclusive: pass one or the other, not both. - `--model ID`: single-model passthrough. Any model id from `GET /v1/models` is accepted; every request is rewritten to `model=ID` and forwarded upstream. - `--routing-profiles PATH`: loads a multi-chain YAML bundle; the first declared route becomes the initial model. -For legacy route-bundle `type: deterministic` routes, the `classifier:` block also accepts: +For `type: deterministic` routes, the `classifier:` block also accepts: | Key | Purpose | |---|---| @@ -85,7 +83,8 @@ For legacy route-bundle `type: deterministic` routes, the `classifier:` block al Benchmark runs started through `benchmark/run-baseline.sh --routing-profiles` record the effective classifier prompt, prompt SHA-256, `max_request_chars`, and `recent_turn_window` in `run_manifest.json` under `server.classifier_prompts`. -**Selecting a v2 profile by id.** A `serve --config` proxy exposes every profile id (and every target id) as a model on `GET /v1/models`. There is no separate "profile" flag. A client, or a launcher using `--model `, selects a profile simply by naming its id as the model. This is the v2 counterpart to picking a route from a bundle. This selection pattern is specific to v2 profile configs; legacy route-bundle configs still use route names as model IDs. +Each route name becomes a model ID on `GET /v1/models`. Clients select a route +by setting the request's `model` field to that name. ### Intake sink (serve and launchers) @@ -147,7 +146,7 @@ matches the pre-1.0 trace format: ``` Turns without an assistant choice (e.g. upstream errors) are skipped. The flag -works on `launch` and on `serve` with a route bundle; `serve --config` rejects it. +works on `launch` and on `serve` with a route bundle. ### Transport (server verbs) @@ -204,14 +203,15 @@ because the built-in default is `https://openrouter.ai/api/v1`. ## `switchyard serve` -Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Rust-defined and Python-defined profiles run through the same Python HTTP application and expose the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id. +Serve a long-running proxy from a YAML route bundle. Each entry under `routes:` +builds a runnable chain and is exposed as a model on `GET /v1/models`. The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropic Messages (`/v1/messages`), and OpenAI Responses (`/v1/responses`) APIs on the same host and port. **Synopsis** ```text -switchyard [--routing-profiles PATH] serve [--config PATH] +switchyard [--routing-profiles PATH] serve [--host HOST] [--port PORT] [--workers N] [--reload] [--inbound FORMAT] [--intake-enabled|--enable-intake [INTAKE OVERRIDES]] @@ -221,41 +221,23 @@ switchyard [--routing-profiles PATH] serve [--config PATH] | Flag | Source | |---|---| -| `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) path. Global flag; pass before `serve`. Falls back to the saved path from `switchyard --routing-profiles PATH -- configure`. | -| `--config PATH` | Switchyard v2 profile-config YAML/JSON/TOML entrypoint. This is the primary serve path. Mutually exclusive with `--routing-profiles`. | +| `--routing-profiles PATH` / `-c PATH` | [Routing](#routing) path. Global flag; pass before `serve`. Falls back to the saved path from `switchyard --routing-profiles PATH -- configure`. | | `--host`, `--port`/`-p`, `--reload` | [Transport](#transport-server-verbs). | -| `--inbound FORMAT` | Valid only for legacy route-bundle serve (`--routing-profiles`); `serve --config` actively rejects it with an error. For legacy serve, the flag is a no-op — all request APIs are always registered regardless of the value (accepted for backwards compat only). | +| `--inbound FORMAT` | Backwards-compatible no-op; all request APIs are always registered. | | `--workers` / `-w` | uvicorn worker count. | -| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. Route-bundle serve only; rejected by `serve --config`. | +| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. | | `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-target-url` | [Intake sink](#intake-sink-serve-and-launchers). | **Notes** - `serve` always registers `POST /v1/chat/completions`, `POST /v1/messages`, `POST /v1/responses`, `GET /v1/models`, and `GET /health`. There is no flag to expose just one request API. -- `GET /v1/stats` and `GET /v1/routing/stats` are available on both serve paths. -- The deprecated route-bundle path accepts `--inbound` for compatibility but ignores it; all supported request APIs are always registered. -- `serve --config` does not support `--reload`, `--workers > 1`, Intake options, `--enable-rl-logging`, `--routing-log-file`, or any explicit `--inbound` value. -- Rust-defined and Python-defined profiles use the same profile-config schema. The profile `type` decides which implementation builds that profile. -- Python-defined profiles are registered via `@profile_config`; the shipped `header-routing` profile is an example. A config can mix a Rust-defined profile and a Python-defined profile, and both profile ids are routable on the same served host and port. +- `GET /v1/stats` and `GET /v1/routing/stats` expose route statistics. +- `--inbound` is retained as a compatibility no-op; all supported request APIs are always registered. **Examples** ```bash -# v2 profile config (primary): each profile id + target id is a model on /v1/models -switchyard serve --config examples/profiles.yaml --port 4000 -# select a profile by id (the `model` field): -curl localhost:4000/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{"model": "smart-stage-router", "messages": [{"role": "user", "content": "hi"}]}' - -# mixed Rust/Python profile config on the same request APIs -switchyard serve --config examples/python_profile.yaml --port 4000 -curl localhost:4000/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -H 'x-switchyard-tier: strong' \ - -d '{"model": "smart", "messages": [{"role": "user", "content": "hi"}]}' - -# Legacy route bundle on port 4000 +# Route bundle on port 4000 switchyard --routing-profiles routes.yaml -- serve --port 4000 # Persist a bundle for later runs; no-TTY configure requires explicit provider credentials @@ -264,7 +246,7 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \ --base-url https://openrouter.ai/api/v1 --no-tui --no-model-discovery switchyard serve --port 4000 -# Multi-worker uvicorn (route-bundle path) +# Multi-worker uvicorn SWITCHYARD_WORKERS=4 switchyard --routing-profiles routes.yaml -- serve ``` diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 4c6e768a..739c70a7 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -1,98 +1,89 @@ # Core Concepts -This page explains the vocabulary the rest of the documentation takes for -granted. Read it once and the configuration examples elsewhere will make sense. -It is not a setup guide: to install and run Switchyard, see -[Getting Started](getting_started.md), and to follow a request through the -system, see [Architecture](architecture.md). +This page explains the vocabulary used throughout the documentation. For setup, +see [Getting Started](getting_started.md). For the request lifecycle, see +[Architecture](architecture.md). ## How it fits together -Switchyard is a proxy. Clients (coding agents, SDKs, your own services) talk to -it on one side, and model backends (hosted providers, private endpoints, local -models) sit on the other. A client never asks for a backend by name. It sends a -model ID, and the configuration behind that ID decides what actually runs. +Switchyard is a proxy. Clients such as coding agents and SDKs talk to it on one +side, while hosted providers, private endpoints, and local model servers sit on +the other. A client sends a model ID, and the route registered under that ID +decides what runs. ```mermaid flowchart LR client["Client
picks a model ID"] - profile["Profile
routing policy"] + route["Route
routing policy"] target["Target
upstream model"] - endpoint["Endpoint
provider connection"] backend["Model backend"] - client -->|"model field"| profile - profile -->|"selects"| target - target -->|"uses"| endpoint - endpoint --> backend + client -->|"model field"| route + route -->|"selects"| target + target --> backend ``` -## Endpoints, targets, and profiles +## Routes and targets -A standalone deployment is described by a profile config: one file with three -sections that keep provider connectivity, upstream models, and client-facing -policy apart. Three terms carry most of the weight. +The Python CLI loads a YAML bundle with a `routes:` section. Each route has a +client-facing name and a `type` that chooses its behavior. Depending on that +type, the route owns one target, two routing tiers, or a larger endpoint list. -| Term | What it is | YAML section | -|---|---|---| -| Endpoint | A provider connection: a `base_url` and its credentials. Many targets can share one. | `endpoints:` | -| Target | A single upstream model you can call. It names an endpoint, a `model`, and a wire `format`. | `targets:` | -| Profile | A client-facing routing policy over one or more targets. Its `type` is the routing strategy. | `profiles:` | - -The three nest. An endpoint is reused by targets, and targets are referenced by -profiles. You set credentials in one place, add a model once, and build as many -policies on top as you need. The [Routing Overview](routing_algorithms/overview.md) -has a complete, runnable config and the full schema. +Shared `defaults:` can provide a base URL, API key, and backend format without +repeating them in every target. The [Routing Overview](routing_algorithms/overview.md) +has a complete runnable bundle. ## Model IDs -Clients choose what happens through the `model` field on a request. Switchyard -registers three kinds of model IDs and lists them all on `GET /v1/models`: -profile IDs, which apply a routing policy; target IDs, which skip routing and -call that one model; and the upstream model name itself, registered as an alias -whenever it differs from its target ID. Send a profile ID when you want routing, -or a target or upstream name when you want to pin a single model. +Every route name is registered as a model ID and listed on `GET /v1/models`. +Clients select a route by putting that name in the request's `model` field. +Some route types also discover or register direct upstream model IDs. ## Tiers and routing strategies Most routing strategies divide traffic between two tiers: a strong target that is more capable and more expensive, and a weak target that is cheaper and -faster. A tier is a role you hand to a target rather than a fixed property of it, -so the same target can be the strong tier in one profile and the weak tier in -another. - -A profile's `type` sets the strategy. `passthrough` sends everything to one -target with no routing. `random-routing` splits traffic on a fixed probability. -`llm-routing` asks a classifier model to pick a tier for each turn. `stage_router` -escalates from weak to strong when request signals call for it. The -[Routing Overview](routing_algorithms/overview.md) covers when to use each and -how to tune it. - -!!! warning "There is no `type: model`" - The current profile config does not have a `type: model`. To expose a single - model, point clients at a target ID directly, or add a `passthrough` profile - when you want a second name for it. `type: model` survives only in the - deprecated `--routing-profiles` bundles used by the launcher compatibility - path. - -Session affinity, or sticky routing, pins a conversation to one tier so later -turns reuse it instead of being classified again. It belongs to `llm-routing` -and is not a strategy of its own; random and stage-router routing decide every -request on its own merits. [Sticky Routing](routing_algorithms/sticky_routing.md) -covers it in full. +faster. A tier is a role assigned inside a route, not a fixed property of a +model. + +A route's `type` sets the strategy. `model` and `passthrough` call one target. +`random_routing` splits traffic on a fixed probability. `deterministic` asks a +classifier model to pick a tier. `stage_router` uses agent-progress signals, +with an optional classifier fallback. `escalation_router` starts on the weak +tier and uses an LLM judge to escalate when needed. The +[Routing Overview](routing_algorithms/overview.md) explains when to use each. + +Session affinity pins a conversation to one tier so later turns reuse it +instead of being classified again. It belongs to deterministic classifier +routing and is not a strategy of its own. Random and stage-router routing make +a decision for every request. [Sticky Routing](routing_algorithms/sticky_routing.md) +covers affinity in full. ## Formats and translation -Clients reach Switchyard in one of three inbound formats: OpenAI Chat -Completions, Anthropic Messages, or OpenAI Responses. Each target has its own -backend format, set by its `format:` field, which is one of `openai`, -`anthropic`, `responses`, or `auto`. When the two differ, Switchyard translates -the request on the way out and the response on the way back. +Clients reach Switchyard through OpenAI Chat Completions, Anthropic Messages, +or OpenAI Responses. Each target has a backend format: `openai`, `anthropic`, +`responses`, or `auto`. When the inbound and backend formats differ, +Switchyard translates the request on the way out and the response on the way +back. + +That translation lets Claude Code, which speaks Anthropic Messages, run against +an OpenAI-compatible model. The [Architecture](architecture.md) page documents +the supported formats and request lifecycle. + +## Programmatic Python profiles + +Embedded Python callers can construct typed profile configs and runtimes +directly instead of loading a route bundle. `ProfileInput` carries the request +and request metadata, while the profile protocols define `run`, `process`, and +`rprocess`. This programmatic API is separate from CLI configuration. + +## Rust server configuration -That translation is what lets Claude Code, which speaks Anthropic Messages, run -against an OpenAI-compatible model, and the reverse. The -[Architecture](architecture.md) page documents every backend format, the `auto` -probe, and the neutral representation used to convert between them. +The `switchyard-server` binary is built directly on libsy. Its TOML file +explicitly defines LLM clients, targets, and algorithm routes; it does not load +Python route bundles. See the +[Rust server README](../crates/switchyard-server/README.md) for details. ## Where to go next diff --git a/docs/guides/agent_launchers.md b/docs/guides/agent_launchers.md index 50e86019..a48ee900 100644 --- a/docs/guides/agent_launchers.md +++ b/docs/guides/agent_launchers.md @@ -28,13 +28,12 @@ switchyard launch openclaw Each launcher reads saved provider credentials from `~/.config/switchyard/`. Interactive Claude Code and Codex sessions show live request and token totals -in the status footer. A saved legacy route bundle is still honored for -compatibility, but route bundles and `--routing-profiles` are deprecated. +in the status footer. A saved route bundle is used when configured. ## Default routing behavior The built-in LLM-as-classifier route is the default only when there is no -`--model`, no CLI `--routing-profiles`, no saved legacy route bundle, and no +`--model`, no CLI `--routing-profiles`, no saved route bundle, and no saved model default. Each turn is classified and dispatched to a weak or strong tier using the validated coding-agent trio: @@ -46,8 +45,8 @@ strong tier using the validated coding-agent trio: `--weak-model`, `--classifier-model`, `--profile`, and `--classifier-min-confidence` tune only this built-in route. They are ignored -when route resolution selects an explicit or saved legacy route bundle or -single model. +when route resolution selects an explicit or saved route bundle or single +model. ## Override the default route @@ -57,8 +56,8 @@ Use `--model` for a single-model passthrough session: switchyard launch claude --model openai/gpt-4o-mini ``` -For launcher-owned legacy route-bundle routing, whether the bundle contains one -route or several, use: +For launcher route-bundle routing, whether the bundle contains one route or +several, use: ```bash switchyard --routing-profiles routes.yaml -- launch claude @@ -72,12 +71,8 @@ For route-bundle sessions, the footer reports active-model and aggregate request/token counts, including errors. `/v1/routing/stats` aggregates usage across the registered chains. -!!! note "Profile-config boundary" - `--config profiles.yaml` belongs to `switchyard serve` and is the primary - profile configuration path. Launcher subcommands do not accept `--config` - today. Use the deprecated `--routing-profiles` flag only for launcher-owned - legacy route-bundle routing; use `switchyard serve --config profiles.yaml` - for standalone deployments. +The same `--routing-profiles` bundle can be served directly with +`switchyard --routing-profiles routes.yaml -- serve`. ## Model requirements @@ -143,7 +138,7 @@ always has either an existing `claude`/`anthropic` prefix or the generated This aliasing applies only to the Claude launcher. `switchyard launch codex`, `switchyard launch openclaw`, and `switchyard serve` expose route ids verbatim. -You can also write the `claude-` prefix directly in legacy route YAML if you +You can also write the `claude-` prefix directly in route YAML if you want the route ids to match exactly what appears in `/model`: ```yaml diff --git a/docs/index.md b/docs/index.md index fd07e221..3801dd0a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,42 +81,33 @@ For source installs, non-interactive configuration, and a curl sanity check, use ## Configuration Model -Standalone deployments start with a profile config that separates provider -connectivity, upstream targets, and client-facing profiles: +Python CLI deployments define client-facing routes in a YAML bundle: ```yaml -endpoints: - openrouter: - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - -targets: - strong: - endpoint: openrouter - model: openai/gpt-4o - format: openai - weak: - endpoint: openrouter - model: openai/gpt-4o-mini - format: openai - -profiles: +defaults: + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + +routes: smart: - type: random-routing - strong: strong - weak: weak + type: random_routing + strong: + model: openai/gpt-4o + weak: + model: openai/gpt-4o-mini strong_probability: 0.3 + fallback_target_on_evict: strong ``` -Run it as a long-lived proxy. Profile and target ids appear as models on -`GET /v1/models`, and clients select one with the request's `model` field: +Run it as a long-lived proxy. Route names appear as models on `GET /v1/models`, +and clients select one with the request's `model` field: ```bash -switchyard serve --config profiles.yaml --port 4000 +switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` -The deprecated `--routing-profiles` flag is retained only for launcher-owned -legacy bundles and saved bundle paths: +The same bundle can drive a launcher or be saved as the default: ```bash switchyard --routing-profiles routes.yaml -- launch claude @@ -128,8 +119,9 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \ Non-interactive `configure` does not read provider credentials from the routing bundle; pass `--api-key` explicitly when persisting the bundle for CI. -Profile ids, direct targets, legacy launcher compatibility, and persistence are -covered in [Routing Overview](routing_algorithms/overview.md). +Route types, launcher use, and persistence are covered in +[Routing Overview](routing_algorithms/overview.md). The Rust server uses its +own explicit TOML configuration for LLM clients, targets, and libsy algorithms. ## Routing Reference diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md index 9df6ffac..d57b731b 100644 --- a/docs/internal/latency_service_routing.md +++ b/docs/internal/latency_service_routing.md @@ -275,10 +275,6 @@ policy-specific CLI flags. Run it with: switchyard --routing-profiles my_routes.yaml -- serve --port 4100 ``` -The Rust `type: latency-service` profile loaded by `switchyard serve --config` -is a separate schema and does not yet expose `session_affinity` or -`affinity_max_sessions`. - ### Config schema `LatencyServiceBackendConfig` diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index 3dcdc18f..cdba2a31 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -14,11 +14,8 @@ a drop-in scrape config and starter alert rules. | Auth | None. Designed as a public scrape endpoint | | Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) | -!!! note "Availability: legacy route-bundle serve only" - `GET /metrics` is served by the **legacy route-bundle path** - (`switchyard --routing-profiles PATH serve`). The v2 profile-config path - (`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats` - (or its alias `GET /v1/routing/stats`) on that path instead. +`GET /metrics` is served by the Python route-bundle server started with +`switchyard --routing-profiles PATH -- serve`. A JSON variant of the same underlying data lives at `GET /v1/stats`, with `GET /v1/routing/stats` as a backwards-compatible alias. diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index d92fd36a..696512c3 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -71,10 +71,8 @@ without clearing an existing confirmation streak. ## Configure an escalation route -Escalation routing is currently available through the legacy `routes:` bundle -used by `--routing-profiles`. This path remains necessary for launcher-owned -routing; route bundles and `--routing-profiles` are otherwise deprecated in -favor of `switchyard serve --config` profiles. +Configure escalation routing in the `routes:` bundle loaded by +`--routing-profiles`. ```yaml routes: diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 7993ab53..f513589c 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -12,66 +12,50 @@ policies default to `strong`. ## Choose a policy -Set `profile_name` for the traffic you expect: +Set `profile` for the traffic you expect: -| `profile_name` | Use for | Default tier mapping | +| `profile` | Use for | Default tier mapping | |---|---|---| | `general` | Mixed chat or API traffic | `simple` uses `weak`; all higher tiers use `strong`. | | `coding_agent` | Claude Code, Codex, Cursor-style agents | `simple` and `medium` use `weak`; `complex` and `reasoning` use `strong`. Tool-planning turns can escalate. | | `openclaw` | OpenClaw personal-assistant traffic | `simple` and `medium` use `weak`; `complex` and `reasoning` use `strong`. Tool orchestration and high-risk external actions can escalate. | -For coding-agent traffic, start with `profile_name: coding_agent`. +For coding-agent traffic, start with `profile: coding_agent`. -## Configure a classifier profile +## Configure a classifier route -Define the strong, weak, and classifier models as targets, then reference those -target IDs from an `llm-routing` profile: +Define the strong, weak, and classifier models in a `deterministic` route: ```yaml -endpoints: - openrouter: - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - -targets: - strong: - endpoint: openrouter - model: openai/gpt-4o - format: openai - weak: - endpoint: openrouter - model: openai/gpt-4o-mini - format: openai - classifier: - endpoint: openrouter - model: nvidia/nemotron-3-nano-30b-a3b - format: openai - extra_body: - reasoning: - enabled: false - -profiles: +defaults: + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + +routes: smart: - type: llm-routing - profile_name: coding_agent - strong: strong - weak: weak - classifier: classifier + type: deterministic + profile: coding_agent + classifier: + model: nvidia/nemotron-3-nano-30b-a3b + min_confidence: 0.6 + fail_open: true + recent_turn_window: 4 + strong: + model: openai/gpt-4o + weak: + model: openai/gpt-4o-mini fallback_target_on_evict: strong - classifier_min_confidence: 0.6 - classifier_fail_open: true - classifier_recent_turn_window: 4 ``` -The classifier target must use `format: openai`. Start the profile server with: +Start the server with: ```bash -switchyard serve --config profiles.yaml --port 4000 +switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` -The profile ID (`smart`) is the model ID clients select for classifier-based -routing. The target IDs remain directly selectable when a client needs to -bypass the classifier. +The route ID (`smart`) is the model ID clients select for classifier-based +routing. Try the profile with representative requests: @@ -92,12 +76,11 @@ prompt determine the verdict. ## Useful options -| Option | Use it when | +| Configuration path | Use it when | |---|---| -| `classifier_min_confidence` | Low-confidence results should use `default_tier` instead of the classifier policy. | -| `classifier_fail_open` | Classifier errors should use `default_tier` rather than fail the client request. | -| `classifier_recent_turn_window` | The classifier needs more or less recent conversation and tool context. | -| `classifier_max_tokens` | You need to cap the classifier tool-call response. | +| `classifier.min_confidence` | Low-confidence results should use the default tier instead of the classifier policy. | +| `classifier.fail_open` | Classifier errors should use the default tier rather than fail the client request. | +| `classifier.recent_turn_window` | The classifier needs more or less recent conversation and tool context. | | `alignment_min_confidence` | A classifier recommendation should only raise the policy tier above this confidence. | | `default_tier` | Abstain, low-confidence, and fail-open decisions should use a tier other than the default `strong`. | | `tier_mapping` | The four classifier policy tiers need a custom mapping to `weak` or `strong`. | @@ -115,10 +98,8 @@ store between the classifier and tier selector. After any configured reuse that tier before classification, so they skip the classifier call; abstain, low-confidence, missing-signal, and fail-open decisions do not pin. -The CLI currently exposes these fields on a `type: deterministic` entry in a -`routes:` bundle loaded with `--routing-profiles`. The Rust `llm-routing` -profile loaded by `switchyard serve --config` does not yet expose them. See -[Session Affinity](sticky_routing.md) for YAML and +Configure these fields on the `type: deterministic` entry in the `routes:` +bundle. See [Session Affinity](sticky_routing.md) for YAML and [How session affinity composes](overview.md#how-session-affinity-composes) for the interaction with routing decisions. diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index dc7440c3..28ff6b35 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -1,198 +1,113 @@ # Routing Overview -Switchyard profile configs register profiles and targets as model IDs that -clients can select through OpenAI Chat Completions, Anthropic Messages, or -OpenAI Responses API requests. Start a profile config with: +The Python CLI loads routing policies from a YAML bundle. Each key under +`routes:` becomes a model ID available through OpenAI Chat Completions, +Anthropic Messages, and OpenAI Responses requests: ```bash -switchyard serve --config profiles.yaml --port 4000 +switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` -Use this page to choose a routing strategy first, then open its detailed page -for configuration and tuning. +Use this page to choose a routing strategy, then open its detailed page for +configuration and tuning. ## Choose a strategy -| Strategy | Use it when | Profile `type` | +| Strategy | Use it when | Route `type` | |---|---|---| -| [Random Routing](random_routing.md) | You need a fixed strong/weak split for A/B tests, baselines, or cost experiments. | `random-routing` | -| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm-routing` | +| [Random Routing](random_routing.md) | You need a fixed strong/weak split for A/B tests, baselines, or cost experiments. | `random_routing` | +| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `deterministic` | | [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` | -| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and let an LLM judge escalate to strong — one-way per task — when the run is in trouble. | `escalation_router` (`routes:` bundle) | +| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `escalation_router` | [Session Affinity (Sticky Routing)](sticky_routing.md) is an opt-in feature of -LLM classifier routing, not a standalone routing strategy. The classifier -integrates the pin into its decision path. See -[How session affinity composes](#how-session-affinity-composes) for the exact -behavior. +LLM classifier routing, not a standalone routing strategy. -## Common profile shape +## Common route shape -Profile configs separate provider connectivity, upstream targets, and -client-facing profiles: +Provider defaults can be shared across all routes, while each route owns its +target configuration: ```yaml -endpoints: - openrouter: - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 - -targets: - strong: - endpoint: openrouter - model: openai/gpt-4o - format: openai - weak: - endpoint: openrouter - model: openai/gpt-4o-mini - format: openai +defaults: + api_key: ${OPENROUTER_API_KEY} + base_url: https://openrouter.ai/api/v1 + format: openai + +routes: + fast: + type: model + target: openai/gpt-4o-mini -profiles: smart: - type: random-routing - strong: strong - weak: weak + type: random_routing + strong: + model: openai/gpt-4o + weak: + model: openai/gpt-4o-mini strong_probability: 0.3 + fallback_target_on_evict: strong ``` -The profile ID (`smart`) is the model ID clients send when they want the -routing policy. Target IDs (`strong` and `weak`) are also directly selectable. -When an upstream model ID differs from its target ID, the profile server -registers that model ID as an additional direct alias. +Use the route name (`fast` or `smart`) as the request's model ID. A single +bundle can serve multiple routes on the same host and port. The examples use model IDs from the [OpenRouter model catalog](https://openrouter.ai/api/v1/models). Select IDs available to your account before deploying; catalog availability can change. -## Multiple profiles - -A single file can declare multiple profiles over the same targets. Each -profile and target appears on `GET /v1/models`: - -```yaml -profiles: - fast: - type: passthrough - target: weak - - smart: - type: random-routing - strong: strong - weak: weak - strong_probability: 0.3 -``` - -Use the profile ID to select policy behavior (`fast` or `smart`) and a target ID -to bypass routing (`weak` or `strong`). +## Model and passthrough routes -## Sub-agent override (`subagent_target`) +- `type: model` registers one explicit model alias without model discovery. +- `type: passthrough` queries the upstream model catalog and registers the + discovered models. -Any profile can name an optional `subagent_target` in its common envelope, -alongside `type`. When a request carries a recognized sub-agent signal — -Codex delegated-work kinds (`x-openai-subagent: collab_spawn` or `review`), -Claude Code agent lineage headers, or an explicit -`x-switchyard-is-subagent: true` — it bypasses the profile's routing and runs -as a direct passthrough to that target: - -```yaml -profiles: - smart: - type: random-routing - strong: strong - weak: weak - strong_probability: 0.3 - subagent_target: weak -``` - -This keeps a sub-agent loop on one intentional, cache-compatible target -instead of re-routing every worker turn. Codex harness-maintenance turns -(`compact`, `memory_consolidation`) and unrecognized kinds stay on normal -profile routing, as does everything else when the field is absent. An unknown -target reference fails at startup, and a sub-agent target failure surfaces as -a normal target error — it is never silently re-routed through the profile. - -To suppress sub-agent routing for a request that carries a recognized signal, -send `x-switchyard-is-subagent: false`. This explicit header overrides Codex -and Claude Code lineage signals in either direction: `false` keeps the request -on normal profile routing even when delegated-work headers are present, and -`true` marks a request as a sub-agent even when no harness headers appear. - -## Direct targets and passthrough aliases - -For new profile configs, use one public model concept: - -- Select a target ID directly when its configured name is the client-facing - name you want. -- Add a `type: passthrough` profile only when you need another stable alias for - that target, such as `fast` above. - -Both choices call the same single target. There is no `type: model` profile in -the current profile schema. - -The deprecated `--routing-profiles` route-bundle format has a separate legacy -distinction: `type: model` registers one explicit alias without model discovery, -while `type: passthrough` queries the upstream model catalog and registers the -discovered models too. That distinction applies only to legacy `routes:` -bundles used by the launcher compatibility path. +Both create direct, single-target chains. Use a routing policy when requests +must be split or classified across targets. ## Self-hosted targets -Any profile target can point at an OpenAI-compatible model server you operate. +Any route target can point at an OpenAI-compatible model server you operate. For example, start a local vLLM server: ```bash vllm serve ./my-rl-qwen --served-model-name my-rl-qwen --port 8000 ``` -Then declare it as a normal endpoint and target: +Then configure it as a normal route: ```yaml -endpoints: +routes: local: + type: model + target: my-rl-qwen base_url: http://localhost:8000/v1 api_key: dummy - -targets: - local-weak: - endpoint: local - model: my-rl-qwen format: openai - -profiles: - fast-local: - type: passthrough - target: local-weak ``` -Reference `local-weak` from any routing profile field that accepts a target ID, -including `strong`, `weak`, `target`, or `targets`. Switchyard does not start or -manage the model server; it only sends requests to the configured endpoint. +Switchyard does not start or manage the model server; it only sends requests +to the configured endpoint. ## How session affinity composes -Session affinity is configured directly on the LLM classifier router. It is not -a generic wrapper applied after every routing strategy. After the configured -warmup, the first confident policy, tool-planning, or alignment verdict pins -the tier. Abstain, low-confidence, missing-signal, and fail-open decisions never -pin. The classifier and tier selector share one affinity store, and later turns -check that store before classification, reuse the tier, and skip the classifier -call. +Session affinity is configured directly on the LLM classifier router. After +the configured warmup, the first confident policy, tool-planning, or alignment +verdict pins the tier. Abstain, low-confidence, missing-signal, and fail-open +decisions never pin. Later turns reuse the tier and skip the classifier call. Pins use a bounded in-process LRU keyed from the stable conversation prefix. They are not shared across workers or restarts. See [Sticky Routing](sticky_routing.md) for configuration and key derivation. -Random and stage-router routing do not expose session-affinity settings; they -continue to make a routing decision for each request. +Random and stage-router routing make a fresh routing decision for each request. +The escalation router instead uses affinity as a one-way escalation latch: only +the strong tier is pinned, and the judge runs until that latch fires. -The escalation router uses the same affinity store with the opposite policy: -affinity is always on (the pin *is* the escalation latch), only the strong -tier is ever pinned, and once the judge starts (a configurable minimum turn), -it keeps running each judged turn until the latch fires. See -[Escalation-Router Routing](escalation_router_routing.md). +## Rust server configuration -!!! note "CLI schema availability" - The CLI currently accepts these settings in a `deterministic` entry in a - `routes:` bundle loaded with `--routing-profiles`. The Rust `llm-routing` - profile loaded by `switchyard serve --config` does not yet expose - session-affinity fields. +The `switchyard-server` binary has a separate TOML schema that explicitly +constructs LLM clients, targets, and libsy algorithms. It does not load Python +route bundles. See the +[Rust server README](../../crates/switchyard-server/README.md) for its supported +configuration. diff --git a/docs/routing_algorithms/random_routing.md b/docs/routing_algorithms/random_routing.md index 360b1031..d7b85bae 100644 --- a/docs/routing_algorithms/random_routing.md +++ b/docs/routing_algorithms/random_routing.md @@ -30,40 +30,7 @@ tier model instead of the routing model. ## Enable It -Use `serve --config` with a profile config: - -```yaml -targets: - strong: - model: openai/gpt-4o - format: openai - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - weak: - model: openai/gpt-4o-mini - format: openai - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - -profiles: - ab-test: - type: random-routing - strong: strong - weak: weak - strong_probability: 0.3 -``` - -Run it with: - -```bash -switchyard serve --config routes.yaml --port 4000 -``` - -The profile id (`ab-test`) is the model id clients select when they want the -weighted split. - -For routing-profile YAML used by launchers, use `type: random_routing` under -`routes:`: +Use `type: random_routing` under `routes:`: ```yaml defaults: @@ -82,3 +49,12 @@ routes: rng_seed: 42 fallback_target_on_evict: strong ``` + +Run it with: + +```bash +switchyard --routing-profiles routes.yaml -- serve --port 4000 +``` + +The route id (`ab-test`) is the model id clients select when they want the +weighted split. diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index a9ec0238..45c9fe04 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -89,23 +89,14 @@ clears the threshold to escalate to capable. (If you add the optional classifier, sub-threshold turns go to it instead of staying on the default tier.) -**Set `0.5` explicitly.** It's the recommended starting point and is what the -example below uses. The default when you omit the field depends on which config -path you take: - -| Configuration path | Default when omitted | -|---|---| -| Profile config (`switchyard serve --config`) | `0.7` | -| Deprecated route bundle (`--routing-profiles`) | `0.5` | - -Rather than rely on either default, set `confidence_threshold: 0.5` yourself. +**Start with `0.5`.** It is the default and the recommended starting point. | `confidence_threshold` | Include `classifier:` block? | Typical use | |---|---|---| | `0.0` | no | Cost/latency-sensitive. Every signal-based verdict is accepted; no per-turn LLM call. Critical-error signals still escalate to capable. | | `0.5` | no | Recommended starting point. A single strong signal clears the threshold on its own; derived from SWE-Bench Pro calibration. | -| `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. `0.7` is the profile-config default when the field is omitted. | -| `1.0` | yes (required) | Classifier-driven. Equivalent to the legacy `coding_agent` profile. | +| `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. | +| `1.0` | yes (required) | Classifier-driven. Equivalent to classifier-only `coding_agent` routing. | The signal-vs-classifier split is dataset-dependent. Measure it in production via `routing_decisions.stage_router` on `/v1/stats` rather than relying on @@ -187,46 +178,37 @@ In stage-router, the efficient model may inherit partial context from the capabl fresh, so RESCUE is a conservative lower bound. Efficient performs at least as well in stage-router as it does alone. -## Profile configuration +## Route configuration ```yaml -endpoints: - openrouter: - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - -# Targets are named by the model id they serve. -targets: - openai/gpt-4o: - endpoint: openrouter - model: openai/gpt-4o - format: openai - openai/gpt-4o-mini: - endpoint: openrouter - model: openai/gpt-4o-mini - format: openai - -profiles: +defaults: + base_url: https://openrouter.ai/api/v1 + api_key: ${OPENROUTER_API_KEY} + format: openai + +routes: smart-stage-router: type: stage_router - picker: capable_first # or efficient_first - confidence_threshold: 0.5 # recommended; range [0.0, 1.0] - signal_recent_window: 3 # sliding window for recent signal counts - fallback_target_on_evict: openai/gpt-4o # required; see Context-Window Handling - capable: openai/gpt-4o # capable tier target id - efficient: openai/gpt-4o-mini # efficient tier target id - enable_stats: true # default true + picker: capable_first + confidence_threshold: 0.5 + signal_recent_window: 3 + fallback_target_on_evict: strong + strong: + id: strong + model: openai/gpt-4o + weak: + id: weak + model: openai/gpt-4o-mini + enable_stats: true ``` -Save the file as `profiles.yaml` and start it with: +Save the file as `routes.yaml` and start it with: ```bash -switchyard serve --config profiles.yaml --port 4000 +switchyard --routing-profiles routes.yaml -- serve --port 4000 ``` This is the recommended default: routing on tool signals alone, no classifier. -If you omit `confidence_threshold`, the profile-config default of `0.7` applies; -the example sets `0.5` explicitly. `fallback_target_on_evict` is required and must reference one of the declared target ids. See [Context-Window Handling](../operations/context_window.md) for @@ -252,10 +234,7 @@ Give the classifier its own credential or quota bucket where you can. Sharing one provider bucket with the efficient tier adds a request per classified turn and can cause sustained 429s at scale. -**Launcher compatibility.** Launcher subcommands don't accept `--config`. A -launcher-owned stage-router still needs the deprecated `--routing-profiles` path -and its legacy `routes:` schema. Use the same `type: stage_router`, picker, -target, classifier, and explicit `confidence_threshold: 0.5` values there. +The same route bundle works with the launchers. ## Observability @@ -271,9 +250,8 @@ harnesses usually capture this same snapshot to a file (see below). ### Decision-source metadata (stage-router-specific) -The profile counts why each turn was routed the way it was, under -`routing_decisions.stage_router` in the stats JSON. For the `serve --config` -profile the values are: +The route counts why each turn was routed the way it was under +`routing_decisions.stage_router` in the stats JSON: | Source | When | |---|---| @@ -281,11 +259,7 @@ profile the values are: | `dimensions` | The signals crossed `confidence_threshold` and picked the tier. | | `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | | `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured, so the default tier was used. | -| `context_overflow_fallback` | A context-window overflow rerouted the turn to `fallback_target_on_evict`. | - -The deprecated `--routing-profiles` path reports the same sources except -`context_overflow_fallback`, plus `no_signal` for turns that arrive before any -tool-result history exists. +| `no_signal` | The request arrived before any tool-result history existed. | To capture a snapshot for a batch run, redirect the endpoint to a file: @@ -295,9 +269,9 @@ curl -s http://localhost:4000/v1/stats > routing_stats_final.json ## When *not* to use stage-router -- **Single-model deployments.** Use a plain passthrough profile instead. +- **Single-model deployments.** Use a `model` or `passthrough` route instead. - **Probabilistic A/B splits.** Use - [Random Routing](random_routing.md) (`type: random-routing` in profile configs). + [Random Routing](random_routing.md) (`type: random_routing`). The stage-router's signals are wasted on a fixed traffic ratio. - **No tool-result history.** Stage-router needs meaningful tool-call traffic to populate the tool-result signal. For pure chat-completion workloads every diff --git a/docs/routing_algorithms/sticky_routing.md b/docs/routing_algorithms/sticky_routing.md index 0b0feb73..404d798a 100644 --- a/docs/routing_algorithms/sticky_routing.md +++ b/docs/routing_algorithms/sticky_routing.md @@ -24,10 +24,8 @@ Opt in with `session_affinity: true` on the route. `affinity_max_sessions` (default `10000`) caps the number of pinned conversations. These fields are part of the deterministic router configuration, not a -standalone route type. In CLI YAML, configure them under a `deterministic` -entry in a `routes:` bundle loaded with `--routing-profiles`. The Rust -`profiles:` schema loaded by `switchyard serve --config` does not yet expose -them. +standalone route type. Configure them under a `deterministic` entry in the +`routes:` bundle loaded with `--routing-profiles`. For deterministic routes, `affinity_warmup_turns` controls how many initial turns remain non-sticky. The default is `0`, which preserves the historical diff --git a/examples/profiles.yaml b/examples/profiles.yaml deleted file mode 100644 index 018ed7fc..00000000 --- a/examples/profiles.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Switchyard v2 profile config — the primary serve surface. -# -# switchyard serve --config examples/profiles.yaml -# -# Each profile id and each target id is exposed as a model on GET /v1/models, so -# a client (or `switchyard launch claude --model `) selects a profile by id: -# fast passthrough to the efficient model -# smart-stage-router signal-based stage-router routing between the efficient and capable models -# the model ids the targets, addressable directly -# -# All profile types here are Rust-defined, so this file is served by the Rust -# profile server. Files that include Python-defined profiles (see -# examples/python_profile.yaml) use the Python FastAPI adapter. - -endpoints: - openrouter: - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - -# Targets are named by the model id they serve, so the profile below reads as -# "capable = , efficient = ". -targets: - anthropic/claude-opus-4.7: - endpoint: openrouter - model: anthropic/claude-opus-4.7 - format: openai - moonshotai/kimi-k2.6: - endpoint: openrouter - model: moonshotai/kimi-k2.6 - format: openai - -profiles: - fast: - type: passthrough - target: moonshotai/kimi-k2.6 - - smart-stage-router: - type: stage_router - capable: anthropic/claude-opus-4.7 - efficient: moonshotai/kimi-k2.6 - fallback_target_on_evict: anthropic/claude-opus-4.7 - picker: capable_first - confidence_threshold: 0.7 - signal_recent_window: 3 - classifier: - model: google/gemini-3.5-flash - api_key: ${OPENROUTER_API_KEY} - base_url: https://openrouter.ai/api/v1 diff --git a/examples/python_profile.yaml b/examples/python_profile.yaml deleted file mode 100644 index ce60423f..00000000 --- a/examples/python_profile.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Switchyard v2 profile config mixing a Rust profile and a Python-defined one. -# -# Load it in-process with the Python API: -# -# from switchyard import load_profiles -# profiles = load_profiles("examples/python_profile.yaml") -# await profiles["smart"].run(ProfileInput(request, metadata)) -# -# Or serve both profile ids over the same HTTP API paths: -# -# switchyard serve --config examples/python_profile.yaml -# -# `direct` is a Rust passthrough; `smart` is the shipped Python `header-routing` -# profile (switchyard/lib/profiles/header_routing.py), which picks a tier from the -# x-switchyard-tier request header and delegates the call to a Rust target -# backend. Both share the same resolved targets. - -endpoints: - openrouter: - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - -targets: - strong: - endpoint: openrouter - model: anthropic/claude-opus-4.7 - format: openai - weak: - endpoint: openrouter - model: moonshotai/kimi-k2.6 - format: openai - -profiles: - direct: - type: passthrough - target: weak - - smart: - type: header-routing - strong: strong - weak: weak diff --git a/switchyard/__init__.py b/switchyard/__init__.py index ce9bbb55..927385ac 100644 --- a/switchyard/__init__.py +++ b/switchyard/__init__.py @@ -80,7 +80,6 @@ StageRouterProfileConfig, TranslateProfileConfig, build_profile, - load_profiles, profile_config, profile_config_type, ) @@ -178,7 +177,6 @@ def __getattr__(name: str) -> Any: "RandomRoutingProfileConfig", "TranslateProfileConfig", "build_profile", - "load_profiles", "profile_config", "profile_config_type", "AnthropicNativeBackend", diff --git a/switchyard/cli/config/user_config.py b/switchyard/cli/config/user_config.py index d7c13a40..3d370c04 100644 --- a/switchyard/cli/config/user_config.py +++ b/switchyard/cli/config/user_config.py @@ -126,7 +126,7 @@ class UserConfig: """Non-secret Switchyard defaults stored under the user config dir. ``routing_profiles`` is a parsed routing-profile YAML bundle saved by - ``switchyard configure --routing-profiles PATH``. Stored as a parsed + ``switchyard --routing-profiles PATH configure``. Stored as a parsed JSON object inline (not as a path — paths rot when files move); env var references inside the bundle are preserved verbatim and re-expanded on each load. Consumed by ``switchyard serve`` (and as a diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 50b5c4db..82e14254 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -499,7 +499,12 @@ def _parse_route_bundle_dict(raw: object) -> RouteBundle: validation runs inside :func:`build_table_from_bundle` so callers that construct a bundle programmatically still benefit. """ - bundle = _require_mapping(_expand_env(raw), "route bundle") + return _validate_route_bundle_dict(_expand_env(raw)) + + +def _validate_route_bundle_dict(raw: object) -> RouteBundle: + """Validate bundle structure without expanding environment references.""" + bundle = _require_mapping(raw, "route bundle") _validate_allowed_keys(bundle, frozenset({"defaults", "routes"}), "route bundle") defaults = _optional_mapping(bundle.get("defaults", {}), "defaults") _validate_allowed_keys(defaults, _TARGET_DEFAULT_KEYS, "defaults") @@ -1221,8 +1226,8 @@ def _stage_router_switchyard( allowed_keys=_STAGE_ROUTER_CLASSIFIER_KEYS, where=f"{model_id}.classifier", ) - # The deprecated bundle keeps the shared strong/weak tier keys (also used by - # deterministic); map them onto StageRouterConfig's capable/efficient fields. + # The YAML schema shares strong/weak tier keys with deterministic routing; + # map them onto StageRouterConfig's capable/efficient fields. resolved = _route_config(route, target_defaults, ("strong", "weak")) resolved["capable"] = resolved.pop("strong") resolved["efficient"] = resolved.pop("weak") diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py index bbb5576d..5a2d9e26 100644 --- a/switchyard/cli/switchyard_cli.py +++ b/switchyard/cli/switchyard_cli.py @@ -7,8 +7,7 @@ Exposed as the ``switchyard`` console script. Subcommands: - serve Serve a v2 profile config (--config) as model-keyed - profiles. + serve Serve a routing-profile bundle. launch claude Spawn Claude Code against a proxy. Single model (``--model``) or full routing via ``--routing-profiles``. Pair with ``--smoke`` to run a one-shot harness round-trip @@ -26,12 +25,7 @@ Examples:: - # v2 profile config (primary serve path): each profile id + target id - # becomes a model on GET /v1/models, selectable with --model . - switchyard serve --config profiles.yaml --port 4000 - - # Deprecated: --routing-profiles is a global flag - # (use -- to separate from the subcommand) + # --routing-profiles is a global flag (use -- to separate it from the subcommand). switchyard --routing-profiles routes.yaml -- serve --port 4000 switchyard --routing-profiles dev.yaml -- serve --port 4001 switchyard --routing-profiles profiles.yaml -- launch claude @@ -49,10 +43,10 @@ # Forwarding args to the launched tool (second -- after the subcommand) switchyard --routing-profiles profiles.yaml -- launch claude -- --no-auto-approve -Legacy routing policies that used to be top-level CLI verbs (``passthrough``, +Routing policies that used to be top-level CLI verbs (``passthrough``, ``random-routing``, ``latency-service``) and launcher flags (``--routing``, ``--weak-model``, ``--strong-probability``, ``--preset``) -are expressed in deprecated routing-profile YAML files. ``serve`` and +are expressed in routing-profile YAML files. ``serve`` and launchers still parse the YAML into profile-backed runtimes. """ @@ -64,7 +58,7 @@ from dataclasses import dataclass from importlib.metadata import PackageNotFoundError, version from inspect import signature -from typing import Any, cast +from typing import Any from switchyard.cli.command_utils import ( quiet_dependency_loggers as _quiet_dependency_loggers, @@ -107,42 +101,11 @@ _DEFAULT_OPENROUTER_BASE_URL = DEFAULT_OPENROUTER_BASE_URL _CANONICAL_INTAKE_ENABLE_FLAG = "--intake-enabled" _DEPRECATED_INTAKE_ENABLE_FLAG = "--enable-intake" -_DEPRECATED_ROUTING_PROFILES_FLAG = "--routing-profiles" _ARGPARSE_ACTION_SUPPORTS_DEPRECATED = ( "deprecated" in signature(argparse.Action.__init__).parameters ) -def _print_deprecation_warning( - subject: str, - details: tuple[str, ...] = (), -) -> None: - """Print a concise, readable CLI deprecation warning to stderr.""" - print(f"warning: {subject} is deprecated.", file=sys.stderr) - for detail in details: - print(f" {detail}", file=sys.stderr) - - -def _warn_deprecated_routing_profiles() -> None: - _print_deprecation_warning( - _DEPRECATED_ROUTING_PROFILES_FLAG, - details=( - "Prefer v2 profile configs with `switchyard serve --config PATH`.", - "Legacy route bundles still run for now, but this flag will be removed in a future release.", - ), - ) - - -def _warn_deprecated_saved_route_bundle() -> None: - _print_deprecation_warning( - "saved routing-profile bundle", - details=( - "Prefer v2 profile configs with `switchyard serve --config PATH`.", - "Clear the saved bundle with `switchyard --routing-profiles '' -- configure`.", - ), - ) - - class _IntakeEnabledAction(argparse.Action): """Store the normalized Intake enable flag and warn on the deprecated alias.""" @@ -396,15 +359,11 @@ def _cmd_launch_openclaw(args: argparse.Namespace) -> None: def _cmd_serve(args: argparse.Namespace) -> None: - """Serve a v2 profile config or a legacy route bundle.""" + """Serve a routing-profile bundle.""" from switchyard.cli.config.user_config import load_user_config from switchyard.cli.route_bundle import build_route_bundle_table routing_profiles = args.routing_profiles - if args.config: - _cmd_serve_profile_config(args) - return - # Intake sink + local RL trace logging both attach as chain processors; # combine them so a single serve invocation can run either or both. intake_request, intake_response = _resolve_intake_processors(args) @@ -429,14 +388,13 @@ def _cmd_serve(args: argparse.Namespace) -> None: if not saved: raise SystemExit( "serve: no routing-profiles given. Pass --routing-profiles " - "PATH or run `switchyard configure --routing-profiles PATH` " + "PATH or run `switchyard --routing-profiles PATH configure` " "to save one." ) # Saved bundles are stored as parsed dicts, so we can skip the # YAML parse step and feed them straight into the dict-driven # entrypoint. Env-var references inside the dict expand inside # build_route_bundle_table on each run. - _warn_deprecated_saved_route_bundle() table = build_route_bundle_table( saved, pre_routing_request_processors=request_processors, @@ -461,100 +419,6 @@ def _cmd_serve(args: argparse.Namespace) -> None: build_and_serve(args, table, inbound_default="both", strategy_summary=strategy_summary) -def _cmd_serve_profile_config(args: argparse.Namespace) -> None: - """Serve a v2 profile config through the Python HTTP application.""" - if args.routing_profiles: - raise SystemExit( - "serve --config cannot be combined with --routing-profiles; " - "use exactly one config surface." - ) - if args.inbound is not None: - raise SystemExit( - "serve --config always exposes " - "OpenAI, Anthropic, and Responses endpoints; omit --inbound." - ) - if args.reload: - raise SystemExit("serve --config does not support --reload.") - if args.workers != 1: - raise SystemExit("serve --config does not support --workers.") - unsupported_intake = any(( - args.intake_enabled, - args.intake_base_url, - args.intake_workspace, - args.intake_api_key, - args.intake_target_url, - )) - if unsupported_intake: - raise SystemExit("serve --config does not support Intake options yet.") - if getattr(args, "enable_rl_logging", False): - raise SystemExit( - "serve --config does not support --enable-rl-logging. " - "Use serve --routing-profiles for local RL trace logging." - ) - if getattr(args, "routing_log_file", None): - raise SystemExit( - "serve --config does not support --routing-log-file. " - "Use serve --routing-profiles for per-request routing logs." - ) - - table = _profile_config_route_table(args.config) - logger.info( - "Switchyard profile config loaded from %s", - args.config, - ) - build_and_serve(args, table, inbound_default="both") - - -def _profile_config_route_table(config_path: str) -> Any: - from switchyard.lib.profiles import PassthroughProfileConfig, ProfileSwitchyard - from switchyard.lib.profiles.loader import load_profiles_and_targets - from switchyard.lib.route_table import RouteTable - - profiles, targets = load_profiles_and_targets(config_path) - table = RouteTable() - for profile_id, profile in profiles.items(): - _register_profile_config_model( - table, - profile_id, - ProfileSwitchyard(cast(Any, profile)), - kind="profile", - ) - for target_id, target in targets.items(): - target_switchyard = ProfileSwitchyard(PassthroughProfileConfig(target=target).build()) - _register_profile_config_model( - table, - target_id, - target_switchyard, - kind="target", - ) - if target.model != target_id: - _register_profile_config_model( - table, - target.model, - target_switchyard, - kind="target-model", - ) - return table - - -def _register_profile_config_model( - table: Any, - model_id: str, - switchyard: Any, - kind: str, -) -> None: - if model_id in table.registered_models(): - raise SystemExit( - f"serve --config: duplicate public model id {model_id!r} while " - "registering profile config routes." - ) - table.register( - model_id, - switchyard, - metadata={"switchyard": {"source": "profile-config", "kind": kind}}, - ) - - # --------------------------------------------------------------------------- # Subcommand: verify {proxy,claude,codex} # --------------------------------------------------------------------------- @@ -847,7 +711,7 @@ def _build_parser() -> argparse.ArgumentParser: default=None, metavar="PATH", help=( - "Deprecated path to a routing-profiles YAML file. Applies to " + "Path to a routing-profiles YAML file. Applies to " "serve, launch, and configure (saves it as the default). Separate " "from the subcommand with -- for clarity: " "switchyard --routing-profiles dev.yaml -- launch claude" @@ -879,24 +743,14 @@ def _build_parser() -> argparse.ArgumentParser: # -- serve -- serve = subparsers.add_parser( "serve", - help="Serve a v2 profile config (--config)", + help="Serve a routing-profile bundle", description=( - "Serve a Switchyard v2 profile config via serve --config: one " - "YAML/JSON/TOML file of endpoints, targets, and profiles; each " - "profile id and target id is exposed on GET /v1/models." + "Serve a routing-profile YAML bundle selected with the global " + "--routing-profiles option." ), ) add_transport_args(serve) _add_intake_args(serve) - serve.add_argument( - "--config", - dest="config", - default=None, - metavar="PATH", - help=( - "Path to a Switchyard v2 profile config (YAML, JSON, or TOML)." - ), - ) serve.add_argument( "--routing-log-file", dest="routing_log_file", @@ -1435,9 +1289,6 @@ def main() -> None: pass args = parser.parse_args(argv) - if args.routing_profiles is not None: - _warn_deprecated_routing_profiles() - if not hasattr(args, "func"): parser.print_help() raise SystemExit(1) diff --git a/switchyard/lib/__init__.py b/switchyard/lib/__init__.py index f4b76bf9..237c5767 100644 --- a/switchyard/lib/__init__.py +++ b/switchyard/lib/__init__.py @@ -11,6 +11,6 @@ - ``translation`` — pure format-conversion functions and typed translation engines - ``processors`` — reusable request/response components used by profiles - ``backends`` — LLM backend implementations (OpenAI, Anthropic, multi-tier routing) -- ``profiles`` — profile-owned runtime/config abstractions for the flatter v2 shape +- ``profiles`` — profile-owned runtime and programmatic config abstractions - ``roles`` — backend role definitions and translation response aliases """ diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py index be52e558..b4faa58b 100644 --- a/switchyard/lib/profiles/__init__.py +++ b/switchyard/lib/profiles/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Python profile abstractions matching the components-v2 design.""" +"""Python profile abstractions for programmatic routing profiles.""" from switchyard.lib.profiles.deterministic_routing_config import ( DeterministicRoutingConfig, @@ -22,7 +22,6 @@ HeaderRoutingProfile, ) from switchyard.lib.profiles.latency_service import LatencyServiceProfileConfig -from switchyard.lib.profiles.loader import load_profiles from switchyard.lib.profiles.noop import NoopProfile, NoopProfileConfig from switchyard.lib.profiles.passthrough import PassthroughProfileConfig from switchyard.lib.profiles.plan_execute import PlanExecuteProfileConfig @@ -86,7 +85,6 @@ "RandomRoutingProfileConfig", "TranslateProfileConfig", "build_profile", - "load_profiles", "profile_config", "profile_config_type", ] diff --git a/switchyard/lib/profiles/header_routing.py b/switchyard/lib/profiles/header_routing.py index f470e360..448017ed 100644 --- a/switchyard/lib/profiles/header_routing.py +++ b/switchyard/lib/profiles/header_routing.py @@ -1,16 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""A header-routing Python profile: pick a tier from a request header. +"""A programmatic Python profile that selects a tier from a request header.""" -A small, real example of a Python-defined v2 profile. It exercises the pieces -the v2 profile system enables: a typed config with ``profile_target`` fields, a -profile-owned processed-request type, and delegation of the actual backend call -through a passthrough profile per target. The architecture mirrors the Rust -profiles in ``switchyard-components-v2``; only the authoring language differs. -""" - -from dataclasses import dataclass, field +from dataclasses import dataclass from switchyard.lib.backends.llm_target import LlmTarget from switchyard.lib.profiles.chain import ComponentChainProfile @@ -29,12 +22,11 @@ class HeaderRoutingConfig: """Route to ``strong`` or ``weak`` based on a request header. A header value of ``strong`` selects the strong target; anything else, or a - missing header, selects ``weak``. Both targets are Rust-owned ``LlmTarget``s - resolved by the shared config loader, exactly like a Rust profile's targets. + missing header, selects ``weak``. """ - strong: LlmTarget = field(metadata={"profile_target": True}) - weak: LlmTarget = field(metadata={"profile_target": True}) + strong: LlmTarget + weak: LlmTarget header: str = _DEFAULT_HEADER def build(self) -> "HeaderRoutingProfile": diff --git a/switchyard/lib/profiles/loader.py b/switchyard/lib/profiles/loader.py deleted file mode 100644 index ed9b2a49..00000000 --- a/switchyard/lib/profiles/loader.py +++ /dev/null @@ -1,216 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Load one v2 profile config into runnable Rust- and Python-defined profiles. - -The YAML/JSON/TOML schema is shared. Rust remains the source of truth for file -format detection, environment interpolation, endpoint inheritance, target -resolution, and Rust-defined profile validation. Python only classifies profile -types registered through :func:`profile_config` and builds those runtimes from -Rust-parsed profile bodies. -""" - -import importlib -from dataclasses import fields, is_dataclass -from pathlib import Path -from typing import Any, cast - -from switchyard.lib.profiles.protocols import ProfileInput, ProfileRunner -from switchyard.lib.profiles.subagent_override import SubagentOverrideProfile -from switchyard.lib.profiles.table import ( - ProfileConfigError, - build_profile, - lookup_profile_config, - registered_profile_config_types, -) -from switchyard_rust.components import LlmTarget -from switchyard_rust.core import ChatResponse -from switchyard_rust.profiles import ( - ProfileConfigPlan, - parse_profile_config_path, -) - -__all__ = ["load_profiles", "load_profiles_and_targets"] - -_PROFILE_TARGET = "profile_target" - - -def load_profiles(path: str | Path) -> dict[str, ProfileRunner]: - """Build every profile in ``path`` into a runnable profile keyed by profile ID. - - Rust-defined profiles (``passthrough``, ``random-routing``, ...) and - Python-defined profiles registered through :func:`profile_config` are built - through the same call, sharing one resolved set of targets. Each returned - value exposes ``async run(input)`` (see :class:`ProfileRunner`). - - Raises :class:`ProfileConfigError` or ``switchyard_rust`` config errors with - profile-level context when the config is invalid. - """ - profiles, _targets = load_profiles_and_targets(path) - return profiles - - -def load_profiles_and_targets( - path: str | Path, -) -> tuple[dict[str, ProfileRunner], dict[str, LlmTarget]]: - """Build profiles and return the resolved target map used by those profiles.""" - _register_shipped_python_profiles() - document: Any = parse_profile_config_path(path) - python_profiles = _python_profiles(document) - plan = document.without_profiles(list(python_profiles)).resolve() - - built: dict[str, ProfileRunner] = {} - for profile_id in plan.profile_ids(): - built[profile_id] = _RustProfileRunner(plan.build_profile(profile_id)) - for profile_id, (profile_type, body) in python_profiles.items(): - built[profile_id] = _build_python_profile(profile_id, profile_type, body, plan) - for profile_id in list(built): - built[profile_id] = _wrap_subagent_override(profile_id, built[profile_id], document, plan) - return built, _targets(plan) - - -def _wrap_subagent_override( - profile_id: str, - profile: ProfileRunner, - document: Any, - plan: ProfileConfigPlan, -) -> ProfileRunner: - """Wrap a built profile when its envelope names a ``subagent_target``. - - Rust profiles already apply :class:`SubagentOverrideProfile` internally at - build time, so Python-level wrapping is skipped for them. Python-defined - profiles receive the Python wrapper here, which adds sub-agent routing - without Rust-level stats (stats for Python profiles remain in the - Python stats pipeline). - - An unknown target reference is a startup configuration error. - """ - target_id = document.profile_subagent_target(profile_id) - if target_id is None: - return profile - # Rust profiles handle subagent routing internally; skip double-wrapping. - if isinstance(profile, _RustProfileRunner): - return profile - target = plan.target(target_id) - if target is None: - raise ProfileConfigError( - f"profile {profile_id}: subagent_target references unknown target {target_id!r}" - ) - from switchyard.lib.profiles.passthrough import PassthroughProfileConfig - - return SubagentOverrideProfile(profile, PassthroughProfileConfig(target=target).build()) - - -class _RustProfileRunner: - """Adapts an erased Rust ``Profile`` to the ``run(input)`` runner contract.""" - - def __init__(self, profile: Any) -> None: - self._profile = profile - - @property - def profile_id(self) -> str: - return cast(str, self._profile.profile_id) - - async def run(self, input: ProfileInput) -> ChatResponse: - return cast(ChatResponse, await self._profile.run(input.request, input.metadata)) - - -def _build_python_profile( - profile_id: str, - profile_type: str, - body: dict[str, Any], - plan: ProfileConfigPlan, -) -> ProfileRunner: - """Construct one Python-defined profile, resolving its target references.""" - config_cls: Any = lookup_profile_config(profile_type) - if not is_dataclass(config_cls): - raise ProfileConfigError( - f"profile {profile_id}: {config_cls.__qualname__} must be a dataclass" - ) - - field_names = {field.name for field in fields(config_cls)} - unknown = sorted(set(body) - field_names - {"type"}) - if unknown: - raise ProfileConfigError( - f"profile {profile_id}: unknown field(s) {unknown} for profile type {profile_type!r}" - ) - - kwargs: dict[str, Any] = {} - for field in fields(config_cls): - if field.name not in body: - continue # let the dataclass default apply - value = body[field.name] - if field.metadata.get(_PROFILE_TARGET): - value = _resolve_targets(profile_id, value, plan) - kwargs[field.name] = value - - # config_cls is a registered dataclass; bind it through Any so mypy does not - # treat it as the (non-instantiable) DataclassInstance protocol. - ctor: Any = config_cls - try: - config = ctor(**kwargs) - except TypeError as exc: # missing required field, wrong arity, ... - raise ProfileConfigError(f"profile {profile_id}: {exc}") from exc - return cast(ProfileRunner, build_profile(config)) - - -def _register_shipped_python_profiles() -> None: - """Import shipped Python profile modules so their config types register.""" - importlib.import_module("switchyard.lib.profiles.header_routing") - - -def _python_profiles( - document: Any, -) -> dict[str, tuple[str, dict[str, Any]]]: - python_types = set(registered_profile_config_types()) - profiles: dict[str, tuple[str, dict[str, Any]]] = {} - for profile_id in document.profile_ids(): - profile_type = document.profile_type(profile_id) - if profile_type is None or profile_type not in python_types: - continue - body = document.profile_body(profile_id) - if not isinstance(body, dict): - raise ProfileConfigError(f"profile {profile_id}: profile body must be a mapping") - profiles[profile_id] = (profile_type, body) - return profiles - - -def _targets(plan: ProfileConfigPlan) -> dict[str, LlmTarget]: - targets: dict[str, LlmTarget] = {} - for target_id in plan.target_ids(): - target = plan.target(target_id) - if target is not None: - targets[target_id] = target - return targets - - -def _resolve_targets( - profile_id: str, value: Any, plan: ProfileConfigPlan -) -> LlmTarget | list[LlmTarget] | None: - """Resolve a ``profile_target`` field's id reference(s) into ``LlmTarget``s. - - The value shape selects the arity, mirroring Rust ``#[profile_target]`` - support for single, optional, and list-valued target fields. - """ - if value is None: - return None - if isinstance(value, str): - return _require_target(profile_id, value, plan) - if isinstance(value, list): - return [_require_target(profile_id, item, plan) for item in value] - raise ProfileConfigError( - f"profile {profile_id}: target reference must be a target id string or " - f"list of ids, got {type(value).__name__}" - ) - - -def _require_target(profile_id: str, target_id: Any, plan: ProfileConfigPlan) -> LlmTarget: - if not isinstance(target_id, str): - raise ProfileConfigError( - f"profile {profile_id}: target reference must be a string, got " - f"{type(target_id).__name__}" - ) - target = plan.target(target_id) - if target is None: - raise ProfileConfigError(f"profile {profile_id}: references unknown target {target_id!r}") - return target diff --git a/switchyard/lib/profiles/protocols.py b/switchyard/lib/profiles/protocols.py index 8d6c5a64..78c4984a 100644 --- a/switchyard/lib/profiles/protocols.py +++ b/switchyard/lib/profiles/protocols.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Python profile protocols matching the components-v2 runtime shape.""" +"""Python-owned profile request values and runtime protocols.""" from typing import Any, ClassVar, Protocol, TypeVar, runtime_checkable @@ -69,11 +69,11 @@ def iter_components(self) -> list[object]: @runtime_checkable class ContextAwareProfile(Profile[ProcessedRequestT], Protocol[ProcessedRequestT]): - """Optional legacy bridge surface for profiles that need ``ProxyContext``. + """Optional component bridge surface for profiles that need ``ProxyContext``. - Pure v2 profiles can implement only ``run`` / ``process`` / ``rprocess``. - Compatibility profiles that still compose existing Python components also - expose these methods so endpoint adapters can reuse the caller's context. + Profiles can implement only ``run`` / ``process`` / ``rprocess``. Profiles + that compose existing Python components also expose these methods so + endpoint adapters can reuse the caller's context. """ async def process_with_context( diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py index ae199d15..ed90e321 100644 --- a/switchyard/lib/profiles/stage_router_config.py +++ b/switchyard/lib/profiles/stage_router_config.py @@ -49,7 +49,7 @@ class StageRouterConfig(BaseModel): #: Scorer confidence in ``[0, 1]`` below which the picker consults the #: classifier (if configured) or returns its default tier. ``0.0`` forces #: pure-deterministic routing; ``1.0`` forces every turn through the - #: classifier (equivalent to the legacy ``coding_agent`` profile). + #: classifier (equivalent to classifier-only ``coding_agent`` routing). confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0) #: Sliding-window size for the Rust signal extractor's ``recent_*`` #: counts (``recent_write_count``, ``recent_edit_count``, etc.). diff --git a/switchyard/lib/profiles/subagent_override.py b/switchyard/lib/profiles/subagent_override.py deleted file mode 100644 index 79493269..00000000 --- a/switchyard/lib/profiles/subagent_override.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Sub-agent override wrapper routing delegated worker requests to a fixed target.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from switchyard.lib.profiles.protocols import ( - ContextAwareProfile, - ProfileLifecycle, - ProfileRunner, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.core import ChatResponse -from switchyard_rust.profiles import ProfileInput, is_subagent_request - - -@dataclass(slots=True) -class SubagentProcessedRequest: - """Pairs the branch selected for one request with its request-side state.""" - - branch: Any - processed: Any - - -class SubagentOverrideProfile: - """Route recognized sub-agent requests to a fixed override branch. - - Wraps a built profile without changing its behavior for normal traffic. A - request whose headers carry a delegated sub-agent signal runs through the - override branch (a passthrough to the profile's configured - ``subagent_target``); every other request runs the wrapped profile - unchanged. The override never rewrites the request or response, and an - override-branch failure surfaces as a normal target error — it is not - re-routed through the wrapped profile. - """ - - def __init__(self, inner: ProfileRunner, override: ProfileRunner) -> None: - """Wrap ``inner``, sending sub-agent requests to ``override`` instead.""" - self._inner = inner - self._override = override - - def iter_components(self) -> list[object]: - """Return lifecycle components of both branches in startup order.""" - return [*_components(self._inner), *_components(self._override)] - - def _branch(self, input: ProfileInput) -> Any: - return self._override if is_subagent_request(input.metadata.headers) else self._inner - - async def run(self, input: ProfileInput) -> ChatResponse: - """Execute the branch selected by the request's sub-agent signal.""" - response = await self._branch(input).run(input) - return response # type: ignore[no-any-return] - - async def run_with_context( - self, - input: ProfileInput, - ctx: ProxyContext, - ) -> ChatResponse: - """Execute the selected branch, preserving the caller-owned context.""" - branch = self._branch(input) - if isinstance(branch, ContextAwareProfile): - return await branch.run_with_context(input, ctx) - response = await branch.run(input) - return response # type: ignore[no-any-return] - - async def process(self, input: ProfileInput) -> SubagentProcessedRequest: - """Run the selected branch's request side, remembering the branch.""" - branch = self._branch(input) - return SubagentProcessedRequest(branch, await branch.process(input)) - - async def process_with_context( - self, - input: ProfileInput, - ctx: ProxyContext, - ) -> SubagentProcessedRequest: - """Run the selected branch's request side with the caller's context.""" - branch = self._branch(input) - if isinstance(branch, ContextAwareProfile): - return SubagentProcessedRequest(branch, await branch.process_with_context(input, ctx)) - return SubagentProcessedRequest(branch, await branch.process(input)) - - async def rprocess( - self, - processed: SubagentProcessedRequest, - response: ChatResponse, - ) -> ChatResponse: - """Run the response side of the branch that processed the request.""" - result = await processed.branch.rprocess(processed.processed, response) - return result # type: ignore[no-any-return] - - -def _components(profile: object) -> list[object]: - """Return a branch's lifecycle components, mirroring ``ProfileSwitchyard``.""" - if isinstance(profile, ProfileLifecycle): - return profile.iter_components() - return [profile] - - -__all__ = ["SubagentOverrideProfile", "SubagentProcessedRequest"] diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py index a7859405..59cbd0a5 100644 --- a/switchyard_rust/core.py +++ b/switchyard_rust/core.py @@ -9,7 +9,6 @@ import importlib.metadata import os from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, ValuesView -from os import PathLike from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, cast JsonScalar: TypeAlias = bool | int | float | str | None @@ -177,47 +176,6 @@ async def call( ) -> Any: ... -class _ProfileConfigDocument(Protocol): - """Rust-owned profile config document exposed through PyO3.""" - - def resolve(self) -> _ProfileConfigPlan: ... - - -class _ProfileConfigPlan(Protocol): - """Rust-owned resolved profile config plan exposed through PyO3.""" - - def profile_ids(self) -> list[str]: ... - def target_ids(self) -> list[str]: ... - def profile_type(self, profile_id: str) -> str | None: ... - def target(self, target_id: str) -> object | None: ... - def build_profile(self, profile_id: str) -> _Profile: ... - def build_profiles(self) -> dict[str, _Profile]: ... - - -class _Profile(Protocol): - """Rust-owned profile runtime exposed through PyO3.""" - - profile_id: str - - async def run(self, request: _ChatRequest) -> _ChatResponse: ... - - -class _ParseProfileConfigStr(Protocol): - def __call__( - self, - input: str, - format: str = "yaml", - ) -> _ProfileConfigDocument: ... - - -class _ParseProfileConfigPath(Protocol): - def __call__(self, path: str | PathLike[str]) -> _ProfileConfigDocument: ... - - -class _LoadProfileConfig(Protocol): - def __call__(self, path: str | PathLike[str]) -> _ProfileConfigPlan: ... - - class _NativeModule(Protocol): ChatRequest: type[_ChatRequest] ChatRequestType: type[_ChatRequestType] @@ -239,15 +197,6 @@ class _NativeModule(Protocol): SwitchyardUpstreamError: type[RuntimeError] SwitchyardContextWindowExceededError: type[RuntimeError] SwitchyardContextPoolExhaustedError: type[RuntimeError] - ProfileConfigDocument: type[_ProfileConfigDocument] - ProfileConfigPlan: type[_ProfileConfigPlan] - Profile: type[_Profile] - load_profile_config: _LoadProfileConfig - parse_profile_config_path: _ParseProfileConfigPath - parse_profile_config_str: _ParseProfileConfigStr - # Shared profile input bindings re-exported through `switchyard_rust.profiles`. - ProfileRequestMetadata: type[Any] - ProfileInput: type[Any] # Session-affinity primitives. SessionCache: type[Any] session_key_from_body: Any diff --git a/switchyard_rust/profiles.py b/switchyard_rust/profiles.py index 7da35c46..aa1dac50 100644 --- a/switchyard_rust/profiles.py +++ b/switchyard_rust/profiles.py @@ -1,37 +1,106 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Components-v2 profile config loading and shared request input bindings. - -Exposes two Rust-owned surfaces: - -* the erased serving API (``ProfileConfigDocument``, ``ProfileConfigPlan``, - ``Profile``) for loading config files and running any profile through - ``run``; and -* shared request input classes (``ProfileInput`` and - ``ProfileRequestMetadata``) used by both Rust and Python profile runtimes. - -Names are resolved lazily from the native extension on first access. -""" - -from switchyard_rust.core import _load_native - -_PROFILE_EXPORTS = ( - # Erased serving surface. - "ProfileConfigDocument", - "ProfileConfigPlan", - "Profile", - "load_profile_config", - "parse_profile_config_path", - "parse_profile_config_str", - "ProfileInput", - "ProfileRequestMetadata", - "is_subagent_request", -) -__all__ = _PROFILE_EXPORTS - - -def __getattr__(name: str) -> object: - if name in _PROFILE_EXPORTS: - return getattr(_load_native(), name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +"""Python-owned request values shared by profile implementations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from switchyard_rust.core import request_type_enum, request_type_value + +if TYPE_CHECKING: + from collections.abc import Mapping + + from switchyard_rust.core import ChatRequest, ChatRequestType + + +@dataclass(frozen=True, slots=True, init=False) +class ProfileRequestMetadata: + """Request metadata available to Python profile runtimes.""" + + request_id: str | None + inbound_format: ChatRequestType | None + headers: dict[str, list[str]] + + def __init__( + self, + request_id: str | None = None, + inbound_format: ChatRequestType | str | None = None, + headers: Mapping[str, str | list[str]] | None = None, + ) -> None: + if request_id is not None and not request_id.strip(): + raise ValueError("invalid request_id: RequestId must not be empty") + object.__setattr__(self, "request_id", request_id) + object.__setattr__( + self, + "inbound_format", + request_type_enum(inbound_format) if inbound_format is not None else None, + ) + object.__setattr__(self, "headers", _normalize_headers(headers)) + + @classmethod + def from_headers( + cls, + headers: Mapping[str, str | list[str]], + inbound_format: ChatRequestType | str | None = None, + ) -> ProfileRequestMetadata: + """Build metadata from headers and infer the request ID when present.""" + normalized = _normalize_headers(headers) + request_ids = normalized.get("x-request-id", []) + return cls( + request_id=request_ids[0] if request_ids else None, + inbound_format=inbound_format, + headers=normalized, + ) + + def to_dict(self) -> dict[str, object]: + """Return a plain dictionary representation.""" + return { + "request_id": self.request_id, + "inbound_format": ( + request_type_value(self.inbound_format) + if self.inbound_format is not None + else None + ), + "headers": {name: list(values) for name, values in self.headers.items()}, + } + + +@dataclass(frozen=True, slots=True, init=False) +class ProfileInput: + """Provider request and metadata passed to a Python profile.""" + + request: ChatRequest + metadata: ProfileRequestMetadata + + def __init__( + self, + request: ChatRequest, + metadata: ProfileRequestMetadata | None = None, + ) -> None: + object.__setattr__(self, "request", request) + object.__setattr__(self, "metadata", metadata or ProfileRequestMetadata()) + + +def _normalize_headers( + headers: Mapping[str, str | list[str]] | None, +) -> dict[str, list[str]]: + normalized: dict[str, list[str]] = {} + for name, value in (headers or {}).items(): + if not isinstance(name, str): + raise TypeError("ProfileRequestMetadata header names must be strings") + if isinstance(value, str): + values = [value] + elif isinstance(value, list) and all(isinstance(item, str) for item in value): + values = list(value) + else: + raise TypeError( + "ProfileRequestMetadata headers must map strings to strings or lists of strings" + ) + normalized.setdefault(name.lower(), []).extend(values) + return normalized + + +__all__ = ["ProfileInput", "ProfileRequestMetadata"] diff --git a/switchyard_rust/profiles.pyi b/switchyard_rust/profiles.pyi index 7c00a0d6..984d7568 100644 --- a/switchyard_rust/profiles.pyi +++ b/switchyard_rust/profiles.pyi @@ -2,41 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Mapping -from os import PathLike -from typing import Literal -from switchyard_rust.components import LlmTarget -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse - -ProfileConfigFormat = Literal["yaml", "yml", "json", "toml"] - -# --- Erased serving surface --------------------------------------------------- - -class ProfileConfigDocument: - def resolve(self) -> ProfileConfigPlan: ... - -class ProfileConfigPlan: - def profile_ids(self) -> list[str]: ... - def target_ids(self) -> list[str]: ... - def profile_type(self, profile_id: str) -> str | None: ... - def target(self, target_id: str) -> LlmTarget | None: ... - def build_profile(self, profile_id: str) -> Profile: ... - def build_profiles(self) -> dict[str, Profile]: ... - -class Profile: - profile_id: str - - async def run(self, request: ChatRequest) -> ChatResponse: ... - -def parse_profile_config_str( - input: str, - format: ProfileConfigFormat = "yaml", -) -> ProfileConfigDocument: ... -def parse_profile_config_path(path: str | PathLike[str]) -> ProfileConfigDocument: ... -def load_profile_config(path: str | PathLike[str]) -> ProfileConfigPlan: ... -def is_subagent_request(headers: Mapping[str, str | list[str]]) -> bool: ... - -# --- Metadata for direct typed profile calls ---------------------------------- +from switchyard_rust.core import ChatRequest, ChatRequestType class ProfileInput: request: ChatRequest @@ -57,12 +24,12 @@ class ProfileRequestMetadata: self, request_id: str | None = None, inbound_format: ChatRequestType | str | None = None, - headers: dict[str, str | list[str]] | None = None, + headers: Mapping[str, str | list[str]] | None = None, ) -> None: ... @classmethod def from_headers( cls, - headers: dict[str, str | list[str]], + headers: Mapping[str, str | list[str]], inbound_format: ChatRequestType | str | None = None, ) -> ProfileRequestMetadata: ... def to_dict(self) -> dict[str, object]: ... diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.py index d39c7de8..6af37699 100644 --- a/tests/readme/test_readme.py +++ b/tests/readme/test_readme.py @@ -7,8 +7,7 @@ * the "Use as a Python library" snippet executes (via ``--markdown-docs`` + the passthrough→noop fixture in ``conftest.py``); -* README route examples validate against the route-bundle schema; -* canonical routing pages still build stage_router and LLM-routing profile examples; +* README and routing-guide examples validate against the route-bundle schema; * every CLI subcommand / flag the README names still exists. """ @@ -23,14 +22,14 @@ from switchyard.cli import route_bundle as rb from switchyard.cli.switchyard_cli import _build_parser -from switchyard_rust.profiles import parse_profile_config_str REPO_ROOT = Path(__file__).resolve().parents[2] README_PATH = REPO_ROOT / "README.md" ROUTING_DOC_PATHS = ( - REPO_ROOT / "docs" / "routing_algorithms" / "stage_router_routing.md", - REPO_ROOT / "docs" / "routing_algorithms" / "llm_classifier_routing.md", REPO_ROOT / "docs" / "routing_algorithms" / "overview.md", + REPO_ROOT / "docs" / "routing_algorithms" / "random_routing.md", + REPO_ROOT / "docs" / "routing_algorithms" / "llm_classifier_routing.md", + REPO_ROOT / "docs" / "routing_algorithms" / "stage_router_routing.md", ) @@ -69,72 +68,67 @@ def test_python_snippet_tripwire(readme_text: str) -> None: ) -def test_all_yaml_route_blocks_in_readme_validate_against_the_schema( - readme_text: str, -) -> None: - # Schema/key validation, not a full chain build: building README's routes +def _validate_route_blocks(text: str, source: Path) -> int: + # Schema/key validation, not a full chain build: building documented routes # is NOT hermetic — `passthrough` with `discover: true` does a live catalog # fetch and `latency_service` polls. The # schema layer (route type + per-type key allowlist) is what we can check # offline, and it catches the likeliest drift: a renamed `type:` or a key # that no longer exists on that type. - blocks = _code_blocks(readme_text, "yaml") - assert blocks, "README unexpectedly has no yaml blocks" - + blocks = _code_blocks(text, "yaml") validated_routes = 0 for idx, block in enumerate(blocks): payload = pyyaml.safe_load(block) if not isinstance(payload, dict) or "routes" not in payload: continue - for model_id, route_raw in payload["routes"].items(): + try: + bundle = rb._validate_route_bundle_dict(payload) + except rb.RouteBundleConfigError as exc: + raise AssertionError( + f"YAML block {idx} in {source} failed bundle schema " + f"validation: {exc}\n\nBlock:\n{block}" + ) from exc + for model_id, route_raw in bundle.routes.items(): route = rb._normalize_route(model_id, route_raw) try: route_type = rb._route_type(model_id, route) rb._validate_route_keys(model_id, route, route_type) except rb.RouteBundleConfigError as exc: raise AssertionError( - f"YAML block {idx} route {model_id!r} in README.md failed " + f"YAML block {idx} route {model_id!r} in {source} failed " f"schema validation: {exc}\n\nBlock:\n{block}" ) from exc validated_routes += 1 + return validated_routes - assert validated_routes, "no README yaml block parsed as a route bundle" + +def test_route_block_validation_rejects_invalid_bundle_defaults() -> None: + text = """```yaml +defaults: + unsupported: true +routes: + noop: + type: noop +``` +""" + with pytest.raises(AssertionError, match="unknown key.*defaults: unsupported"): + _validate_route_blocks(text, Path("example.md")) -def test_stage_router_and_llm_routing_yaml_blocks_in_canonical_docs_build( - monkeypatch: pytest.MonkeyPatch, +def test_all_yaml_route_blocks_in_readme_validate_against_the_schema( + readme_text: str, ) -> None: - monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-readme") - - built_profiles = 0 - for path in ROUTING_DOC_PATHS: - for idx, block in enumerate(_code_blocks(path.read_text(), "yaml")): - payload = pyyaml.safe_load(block) - if not isinstance(payload, dict) or not { - "endpoints", - "targets", - "profiles", - }.issubset(payload): - continue - relevant_profile_ids = [ - profile_id - for profile_id, profile in payload["profiles"].items() - if profile.get("type") in {"stage_router", "llm-routing"} - ] - if not relevant_profile_ids: - continue - try: - plan = parse_profile_config_str(block).resolve() - for profile_id in relevant_profile_ids: - plan.build_profile(profile_id) - except Exception as exc: - raise AssertionError( - f"YAML block {idx} profiles {relevant_profile_ids!r} in {path} " - f"failed profile-config build: {exc}\n\nBlock:\n{block}" - ) from exc - built_profiles += len(relevant_profile_ids) + assert _validate_route_blocks(readme_text, README_PATH), ( + "no README yaml block parsed as a route bundle" + ) + + +def test_canonical_routing_docs_validate_against_the_route_bundle_schema() -> None: + validated_routes = sum( + _validate_route_blocks(path.read_text(), path) for path in ROUTING_DOC_PATHS + ) - assert built_profiles >= 2 + assert validated_routes >= len(ROUTING_DOC_PATHS) def _subparsers(parser: argparse.ArgumentParser) -> dict[str, argparse.ArgumentParser]: diff --git a/tests/test_cli_reference_docs.py b/tests/test_cli_reference_docs.py index 46eebdeb..119e65bc 100644 --- a/tests/test_cli_reference_docs.py +++ b/tests/test_cli_reference_docs.py @@ -99,15 +99,15 @@ def test_cli_reference_omits_unsupported_skill_distillation_flags() -> None: ) -def test_cli_reference_serve_overview_points_to_config_path() -> None: +def test_cli_reference_serve_overview_points_to_route_bundle() -> None: serve_row = next( line for line in CLI_REFERENCE.read_text().splitlines() if line.startswith("| [`serve`](#switchyard-serve)") ) - assert "`serve --config`" in serve_row - assert "--routing-profiles" not in serve_row + assert "--routing-profiles" in serve_row + assert "`serve --config`" not in serve_row assert "inbound format" not in serve_row.lower() diff --git a/tests/test_launch_claude.py b/tests/test_launch_claude.py index 3acc9679..f9b07fbe 100644 --- a/tests/test_launch_claude.py +++ b/tests/test_launch_claude.py @@ -1195,11 +1195,11 @@ def test_errors_when_neither_cli_nor_saved(self, monkeypatch, tmp_path): args = parser.parse_args(["serve", "--port", "4000"]) with pytest.raises(SystemExit) as excinfo: _cmd_serve(args) - assert "routing-profiles" in str(excinfo.value) + assert "switchyard --routing-profiles PATH configure" in str(excinfo.value) class TestConfigurePersistsRoutingProfiles: - """`switchyard configure --routing-profiles PATH` parses + snapshots the bundle.""" + """`switchyard --routing-profiles PATH configure` snapshots the bundle.""" def test_cli_path_persists_parsed_bundle(self, monkeypatch, tmp_path): from switchyard.cli.config.user_config import load_user_config diff --git a/tests/test_profile_migration.py b/tests/test_profile_migration.py deleted file mode 100644 index 58446567..00000000 --- a/tests/test_profile_migration.py +++ /dev/null @@ -1,535 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Migration tests for replacing legacy serving paths with components-v2 profiles.""" - -import errno -import json -import threading -from collections.abc import Iterator -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any - -import pytest -from fastapi.testclient import TestClient -from pytest_mock import MockerFixture - -from switchyard.cli.route_bundle import build_route_bundle_table -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ChatRequest, SwitchyardUpstreamError -from switchyard_rust.profiles import Profile, parse_profile_config_str - - -class _OpenAICompatStub: - def __init__(self) -> None: - self._server: ThreadingHTTPServer | None = None - self._thread: threading.Thread | None = None - self._lock = threading.Lock() - self._requests: list[dict[str, Any]] = [] - self._responses: list[tuple[int, dict[str, object]]] = [] - - def __enter__(self) -> "_OpenAICompatStub": - owner = self - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self) -> None: - length = int(self.headers.get("content-length", "0")) - raw = self.rfile.read(length) - body = json.loads(raw.decode("utf-8")) if raw else {} - with owner._lock: - owner._requests.append({"path": self.path, "body": body}) - if owner._responses: - status, payload = owner._responses.pop(0) - else: - status = 200 - payload = _completion_payload( - str(body.get("model", "missing-model")), self.path, "ok" - ) - - content = json.dumps(payload).encode("utf-8") - self.send_response(status) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(content))) - self.send_header("connection", "close") - self.end_headers() - self.wfile.write(content) - - def log_message(self, _format: str, _line=None, _status=None, _size=None) -> None: - return None - - try: - self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - except OSError as exc: - if exc.errno in {errno.EACCES, errno.EPERM, errno.EADDRNOTAVAIL}: - pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}") - raise - self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) - self._thread.start() - return self - - def __exit__(self, _exc_type: object, _exc: object, _traceback: object) -> None: - if self._server is not None: - self._server.shutdown() - self._server.server_close() - if self._thread is not None: - self._thread.join(timeout=5) - - @property - def base_url(self) -> str: - if self._server is None: - raise RuntimeError("stub server is not running") - host, port = self._server.server_address - return f"http://{host}:{port}" - - @property - def requests(self) -> list[dict[str, Any]]: - with self._lock: - return list(self._requests) - - def respond_json(self, status: int, payload: dict[str, object]) -> None: - with self._lock: - self._responses.append((status, payload)) - - -@pytest.fixture -def openai_stub() -> Iterator[_OpenAICompatStub]: - with _OpenAICompatStub() as stub: - yield stub - - -@pytest.fixture(autouse=True) -def _disable_catalog_discovery(mocker: MockerFixture) -> None: - """Keep legacy route-bundle tests offline and focused on configured routes.""" - mocker.patch( - "switchyard.cli.route_bundle.fetch_model_ids", - return_value=[], - ) - - -def _completion_payload(model: str, path: str, content: str) -> dict[str, object]: - return { - "id": "chatcmpl-profile-migration", - "object": "chat.completion", - "created": 1700000000, - "model": model, - "mock_path": path, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, - } - - -def _tool_call_payload( - model: str, - path: str, - tool_name: str, - arguments: dict[str, object], -) -> dict[str, object]: - return { - "id": "chatcmpl-profile-migration", - "object": "chat.completion", - "created": 1700000000, - "model": model, - "mock_path": path, - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_route", - "type": "function", - "function": { - "name": tool_name, - "arguments": json.dumps(arguments), - }, - } - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, - } - - -def _chat_payload(model: str) -> dict[str, object]: - return {"model": model, "messages": [{"role": "user", "content": "hello"}]} - - -def _legacy_app(route_bundle: dict[str, object]) -> TestClient: - table = build_route_bundle_table(route_bundle) - return TestClient(build_switchyard_app(table), raise_server_exceptions=False) - - -def _profile_runner(config: str, profile_id: str) -> Profile: - return parse_profile_config_str(config).resolve().build_profile(profile_id) - - -def _passthrough_profile_config(base_url: str) -> str: - return f""" -targets: - direct: - model: provider/direct - format: openai - base_url: "{base_url}/v2-direct/v1" - api_key: test-key -profiles: - direct-profile: - type: passthrough - target: direct -""" - - -def _random_profile_config(base_url: str, strong_probability: float) -> str: - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/v2-random/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/v2-random/v1" - api_key: test-key -profiles: - random-profile: - type: random-routing - strong: strong - weak: weak - strong_probability: {strong_probability} - rng_seed: 7 -""" - - -def _latency_profile_config(base_url: str, upstream_path: str = "v2-latency") -> str: - return f""" -targets: - latency: - model: provider/latency - format: openai - base_url: "{base_url}/{upstream_path}/v1" - api_key: test-key -profiles: - latency-profile: - type: latency-service - latency_service_url: "http://latency.invalid" - targets: [latency] - max_retries: 0 -""" - - -def _llm_routing_profile_config(base_url: str) -> str: - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/v2-llm-routing/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/v2-llm-routing/v1" - api_key: test-key - classifier: - model: provider/classifier - format: openai - base_url: "{base_url}/v2-llm-routing-classifier/v1" - api_key: test-key -profiles: - llm-profile: - type: llm-routing - strong: strong - weak: weak - classifier: classifier - profile_name: coding_agent -""" - - -def _stage_router_profile_config(base_url: str) -> str: - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/v2-stage_router/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/v2-stage_router/v1" - api_key: test-key - classifier: - model: provider/classifier - format: openai - base_url: "{base_url}/v2-stage_router-classifier/v1" - api_key: test-key -profiles: - stage_router-profile: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.7 - classifier: - model: provider/classifier - api_key: test-key - base_url: "{base_url}/v2-stage_router-classifier/v1" -""" - - -async def test_passthrough_profile_matches_legacy_single_target_serving( - openai_stub: _OpenAICompatStub, -) -> None: - legacy = _legacy_app( - { - "routes": { - "provider/direct": { - "type": "model", - "target": { - "model": "provider/direct", - "format": "openai", - "base_url": f"{openai_stub.base_url}/legacy-direct/v1", - "api_key": "test-key", - }, - } - } - } - ) - profile = _profile_runner( - _passthrough_profile_config(openai_stub.base_url), - "direct-profile", - ) - - legacy_response = legacy.post( - "/v1/chat/completions", - json=_chat_payload("provider/direct"), - ) - profile_response = await profile.run( - ChatRequest.openai_chat(_chat_payload("direct-profile")), - ) - - assert legacy_response.status_code == 200, legacy_response.text - assert legacy_response.json()["choices"][0]["message"]["content"] == "ok" - assert profile_response.body["choices"][0]["message"]["content"] == "ok" - assert [call["body"]["model"] for call in openai_stub.requests] == [ - "provider/direct", - "provider/direct", - ] - assert [call["path"] for call in openai_stub.requests] == [ - "/legacy-direct/v1/chat/completions", - "/v2-direct/v1/chat/completions", - ] - - -@pytest.mark.parametrize( - ("strong_probability", "expected_model"), - [(1.0, "provider/strong"), (0.0, "provider/weak")], -) -async def test_random_routing_profile_matches_legacy_strong_and_weak_selection( - openai_stub: _OpenAICompatStub, - strong_probability: float, - expected_model: str, -) -> None: - legacy = _legacy_app( - { - "routes": { - "legacy-random": { - "type": "random-routing", - "strong": { - "model": "provider/strong", - "format": "openai", - "base_url": f"{openai_stub.base_url}/legacy-random/v1", - "api_key": "test-key", - }, - "weak": { - "model": "provider/weak", - "format": "openai", - "base_url": f"{openai_stub.base_url}/legacy-random/v1", - "api_key": "test-key", - }, - "fallback_target_on_evict": "strong", - "strong_probability": strong_probability, - "rng_seed": 7, - } - } - } - ) - profile = _profile_runner( - _random_profile_config(openai_stub.base_url, strong_probability), - "random-profile", - ) - - legacy_response = legacy.post( - "/v1/chat/completions", - json=_chat_payload("legacy-random"), - ) - profile_response = await profile.run( - ChatRequest.openai_chat(_chat_payload("random-profile")), - ) - - assert legacy_response.status_code == 200, legacy_response.text - assert profile_response.body["choices"][0]["message"]["content"] == "ok" - assert [call["body"]["model"] for call in openai_stub.requests] == [ - expected_model, - expected_model, - ] - assert [call["path"] for call in openai_stub.requests] == [ - "/legacy-random/v1/chat/completions", - "/v2-random/v1/chat/completions", - ] - - -async def test_latency_service_profile_routes_to_configured_target( - openai_stub: _OpenAICompatStub, -) -> None: - profile = _profile_runner( - _latency_profile_config(openai_stub.base_url), - "latency-profile", - ) - - response = await profile.run( - ChatRequest.openai_chat(_chat_payload("latency-profile")), - ) - - assert response.body["model"] == "provider/latency" - assert [call["path"] for call in openai_stub.requests] == ["/v2-latency/v1/chat/completions"] - assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/latency"] - - -async def test_latency_service_profile_preserves_upstream_error( - openai_stub: _OpenAICompatStub, -) -> None: - openai_stub.respond_json( - 503, - {"error": {"message": "upstream unavailable", "code": "unavailable"}}, - ) - runner = ( - parse_profile_config_str(_latency_profile_config(openai_stub.base_url)) - .resolve() - .build_profile("latency-profile") - ) - - with pytest.raises(SwitchyardUpstreamError, match="upstream unavailable"): - await runner.run(ChatRequest.openai_chat(_chat_payload("latency-profile"))) - - assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/latency"] - - -async def test_llm_routing_profile_routes_with_strict_classifier( - openai_stub: _OpenAICompatStub, -) -> None: - openai_stub.respond_json( - 200, - _tool_call_payload( - "provider/classifier", - "/v2-llm-routing-classifier/v1/chat/completions", - "select_route", - { - "recommended_tier": "medium", - "confidence": 0.9, - "abstain": False, - "turn_type": "exploration", - "code_modification_scope": "none", - "tool_call_count_estimate": 0, - "requires_codebase_context": False, - }, - ), - ) - profile = _profile_runner( - _llm_routing_profile_config(openai_stub.base_url), - "llm-profile", - ) - - response = await profile.run( - ChatRequest.openai_chat(_chat_payload("llm-profile")), - ) - - assert response.body["model"] == "provider/weak" - assert [call["path"] for call in openai_stub.requests] == [ - "/v2-llm-routing-classifier/v1/chat/completions", - "/v2-llm-routing/v1/chat/completions", - ] - assert [call["body"]["model"] for call in openai_stub.requests] == [ - "provider/classifier", - "provider/weak", - ] - classifier_body = openai_stub.requests[0]["body"] - assert classifier_body["tools"][0]["function"]["name"] == "select_route" - assert classifier_body["tools"][0]["function"]["strict"] is True - assert classifier_body["tool_choice"]["function"]["name"] == "select_route" - assert "response_format" not in classifier_body - - -async def test_stage_router_profile_routes_with_classifier( - openai_stub: _OpenAICompatStub, -) -> None: - openai_stub.respond_json( - 200, - _completion_payload( - "provider/classifier", - "/v2-stage_router-classifier/v1/chat/completions", - json.dumps({"tier": "efficient"}), - ), - ) - profile = _profile_runner( - _stage_router_profile_config(openai_stub.base_url), - "stage_router-profile", - ) - - response = await profile.run( - ChatRequest.openai_chat(_chat_payload("stage_router-profile")), - ) - - assert response.body["model"] == "provider/weak" - assert [call["path"] for call in openai_stub.requests] == [ - "/v2-stage_router-classifier/v1/chat/completions", - "/v2-stage_router/v1/chat/completions", - ] - assert [call["body"]["model"] for call in openai_stub.requests] == [ - "provider/classifier", - "provider/weak", - ] - response_format = openai_stub.requests[0]["body"]["response_format"] - assert response_format["type"] == "json_object" - - -def test_python_profile_plan_builds_expected_runner_and_target( - openai_stub: _OpenAICompatStub, -) -> None: - plan = parse_profile_config_str(_passthrough_profile_config(openai_stub.base_url)).resolve() - profiles = plan.build_profiles() - target = plan.target("direct") - - assert plan.profile_ids() == ["direct-profile"] - assert plan.target_ids() == ["direct"] - assert profiles["direct-profile"].profile_id == "direct-profile" - assert target is not None - assert target.model == "provider/direct" - - -async def test_profile_runner_uses_same_profile_config( - openai_stub: _OpenAICompatStub, -) -> None: - runner = _profile_runner( - _passthrough_profile_config(openai_stub.base_url), - "direct-profile", - ) - - response = await runner.run(ChatRequest.openai_chat(_chat_payload("direct-profile"))) - - assert response.body["model"] == "provider/direct" - assert response.body["mock_path"] == "/v2-direct/v1/chat/completions" - assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/direct"] diff --git a/tests/test_python_profile_abstractions.py b/tests/test_python_profile_abstractions.py index c6e9263d..637bb90d 100644 --- a/tests/test_python_profile_abstractions.py +++ b/tests/test_python_profile_abstractions.py @@ -157,6 +157,29 @@ def _request(*, msg: str = "hi") -> ChatRequest: }) +def test_profile_request_metadata_normalizes_headers() -> None: + metadata = ProfileRequestMetadata.from_headers( + { + "X-Request-ID": "req-123", + "X-Switchyard-Trace": ["trace-a", "trace-b"], + }, + inbound_format=ChatRequestType.OPENAI_CHAT, + ) + + assert metadata.request_id == "req-123" + assert metadata.inbound_format == ChatRequestType.OPENAI_CHAT + assert metadata.headers == { + "x-request-id": ["req-123"], + "x-switchyard-trace": ["trace-a", "trace-b"], + } + assert metadata.to_dict()["inbound_format"] == "openai_chat" + + +def test_profile_request_metadata_rejects_empty_request_id() -> None: + with pytest.raises(ValueError, match="RequestId must not be empty"): + ProfileRequestMetadata(request_id=" ") + + async def test_profile_config_decorator_builds_dataclass_and_profile_runtime() -> None: @profile_config("unit-python-static", register=False) class StaticProfileConfig: diff --git a/tests/test_python_profile_loader.py b/tests/test_python_profile_loader.py deleted file mode 100644 index e406bba7..00000000 --- a/tests/test_python_profile_loader.py +++ /dev/null @@ -1,227 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the unified Python/Rust v2 profile loader. - -The headline behavior: one config file builds and runs *both* a Rust-defined -profile and a Python-defined profile through the same loading path -(``load_profiles``), sharing one resolved set of targets. -""" - -import json -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any - -import pytest - -from switchyard import load_profiles -from switchyard.lib.profiles import ProfileConfigError -from switchyard_rust.core import ChatRequest, SwitchyardConfigError -from switchyard_rust.profiles import ProfileInput, ProfileRequestMetadata - - -class _MockOpenAIServer(ThreadingHTTPServer): - calls: list[dict[str, Any]] - - def __init__(self) -> None: - super().__init__(("127.0.0.1", 0), _MockOpenAIHandler) - self.calls = [] - - @property - def base_url(self) -> str: - host, port = self.server_address - return f"http://{host}:{port}" - - -class _MockOpenAIHandler(BaseHTTPRequestHandler): - server: _MockOpenAIServer - - def do_POST(self) -> None: - content_length = int(self.headers.get("content-length", "0")) - raw_body = self.rfile.read(content_length) - body = json.loads(raw_body.decode("utf-8")) if raw_body else {} - self.server.calls.append({"path": self.path, "body": body}) - response = { - "id": "chatcmpl-loader", - "object": "chat.completion", - "model": body.get("model"), - "mock_path": self.path, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, - } - payload = json.dumps(response).encode("utf-8") - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message(self, _format: str, *_args: object) -> None: - return - - -@pytest.fixture -def mock_openai_server() -> _MockOpenAIServer: - try: - server = _MockOpenAIServer() - except PermissionError as exc: - pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}") - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield server - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - -def _mixed_config(base_url: str) -> str: - """A config with a Rust passthrough and the Python header-routing profile.""" - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/strong/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/weak/v1" - api_key: test-key - -profiles: - direct: - type: passthrough - target: weak - smart: - type: header-routing - strong: strong - weak: weak -""" - - -def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path: - path = tmp_path / name - path.write_text(text, encoding="utf-8") - return path - - -def _request(model: str) -> ChatRequest: - return ChatRequest.openai_chat( - {"model": model, "messages": [{"role": "user", "content": "hello"}]} - ) - - -def test_load_profiles_builds_rust_and_python_profiles(tmp_path: Path) -> None: - path = _write(tmp_path, _mixed_config("http://127.0.0.1:9")) - profiles = load_profiles(path) - # One Rust-defined profile (passthrough) and one Python-defined profile - # (header-routing) built through the same path. - assert sorted(profiles) == ["direct", "smart"] - assert all(hasattr(runner, "run") for runner in profiles.values()) - - -async def test_loaded_rust_and_python_profiles_run_against_mock( - mock_openai_server: _MockOpenAIServer, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _mixed_config(mock_openai_server.base_url)) - profiles = load_profiles(path) - - # Rust passthrough -> weak target. - direct = await profiles["direct"].run(ProfileInput(_request("client/x"))) - assert direct.body["model"] == "provider/weak" - assert direct.body["mock_path"] == "/weak/v1/chat/completions" - - # Python header-routing: header selects strong, delegating to the Rust backend. - strong_meta = ProfileRequestMetadata(headers={"x-switchyard-tier": "strong"}) - strong = await profiles["smart"].run(ProfileInput(_request("client/x"), strong_meta)) - assert strong.body["model"] == "provider/strong" - assert strong.body["mock_path"] == "/strong/v1/chat/completions" - - # Missing/other header falls back to weak. - weak = await profiles["smart"].run(ProfileInput(_request("client/x"))) - assert weak.body["model"] == "provider/weak" - assert weak.body["mock_path"] == "/weak/v1/chat/completions" - - -def test_load_profiles_accepts_json(tmp_path: Path) -> None: - document = { - "targets": { - "weak": { - "model": "provider/weak", - "format": "openai", - "base_url": "http://127.0.0.1:9/weak/v1", - "api_key": "test-key", - } - }, - "profiles": {"direct": {"type": "passthrough", "target": "weak"}}, - } - path = _write(tmp_path, json.dumps(document), name="profiles.json") - assert sorted(load_profiles(path)) == ["direct"] - - -def test_unknown_profile_type_is_rejected(tmp_path: Path) -> None: - path = _write( - tmp_path, - "profiles:\n bad:\n type: does-not-exist\n", - ) - with pytest.raises(SwitchyardConfigError, match="does-not-exist"): - load_profiles(path) - - -def test_python_profile_missing_target_is_rejected(tmp_path: Path) -> None: - config = """ -targets: - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:9/weak/v1 - api_key: test-key - -profiles: - smart: - type: header-routing - strong: ghost - weak: weak -""" - path = _write(tmp_path, config) - with pytest.raises(ProfileConfigError, match="unknown target 'ghost'"): - load_profiles(path) - - -def test_python_profile_unknown_field_is_rejected(tmp_path: Path) -> None: - config = """ -targets: - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:9/weak/v1 - api_key: test-key - -profiles: - smart: - type: header-routing - strong: weak - weak: weak - bogus: 1 -""" - path = _write(tmp_path, config) - with pytest.raises(ProfileConfigError, match="unknown field"): - load_profiles(path) - - -def test_unknown_format_is_rejected(tmp_path: Path) -> None: - path = _write(tmp_path, "profiles: {}\n", name="profiles.ini") - with pytest.raises(SwitchyardConfigError, match="unsupported profile config extension"): - load_profiles(path) diff --git a/tests/test_rl_logging.py b/tests/test_rl_logging.py index bb93ecdc..0788413e 100644 --- a/tests/test_rl_logging.py +++ b/tests/test_rl_logging.py @@ -339,7 +339,7 @@ def _fake_load(routing_profiles, *, pre_routing_request_processors=(), monkeypatch.setattr(cli, "build_and_serve", lambda *a, **k: None) args = argparse.Namespace( - config=None, routing_profiles="profiles.yaml", + routing_profiles="profiles.yaml", enable_rl_logging=True, rl_log_dir=str(tmp_path), intake_enabled=False, intake_base_url=None, intake_workspace=None, intake_api_key=None, intake_target_url=None, @@ -348,18 +348,3 @@ def _fake_load(routing_profiles, *, pre_routing_request_processors=(), assert [type(p).__name__ for p in captured["request"]] == ["RlLoggingRequestProcessor"] assert [type(p).__name__ for p in captured["response"]] == ["RlLoggingResponseProcessor"] - - -def test_serve_config_rejects_rl_logging() -> None: - """The Rust profile-server path has no Python chain, so it must reject the flag.""" - from switchyard.cli.switchyard_cli import _cmd_serve_profile_config - - args = argparse.Namespace( - config="profiles.yaml", routing_profiles=None, inbound=None, - reload=False, workers=1, - intake_enabled=False, intake_base_url=None, intake_workspace=None, - intake_api_key=None, intake_target_url=None, - enable_rl_logging=True, rl_log_dir="./rl_data", - ) - with pytest.raises(SystemExit, match="enable-rl-logging"): - _cmd_serve_profile_config(args) diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index cafe39e1..684b25ac 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -421,54 +421,6 @@ def _fake_serve( assert captured["inbound_default"] == "both" -def test_serve_config_and_routing_profiles_are_mutually_exclusive( - tmp_path, -) -> None: - config_path = tmp_path / "profiles.yaml" - routes_path = tmp_path / "routes.yaml" - config_path.write_text("profiles:\n bench:\n type: noop\n") - routes_path.write_text("routes:\n bench:\n type: noop\n") - - args = cli._build_parser().parse_args([ - "--routing-profiles", - str(routes_path), - "serve", - "--config", - str(config_path), - ]) - - with pytest.raises(SystemExit, match="cannot be combined"): - args.func(args) - - -@pytest.mark.parametrize( - "serve_args, match", - [ - (["--reload"], "--reload"), - (["--workers", "2"], "--workers"), - (["--inbound", "openai"], "--inbound"), - (["--inbound", "both"], "--inbound"), - (["--intake-enabled"], "Intake"), - ], -) -def test_serve_config_rejects_python_only_options( - tmp_path, - serve_args: list[str], - match: str, -) -> None: - config_path = tmp_path / "profiles.yaml" - config_path.write_text("profiles:\n bench:\n type: noop\n") - args = cli._build_parser().parse_args([ - "serve", - "--config", - str(config_path), - *serve_args, - ]) - - with pytest.raises(SystemExit, match=match): - args.func(args) - - def test_main_reports_route_bundle_config_error_without_traceback( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -554,7 +506,7 @@ def test_main_reports_non_utf8_route_bundle_without_traceback( assert "\n" not in str(excinfo.value.code) -def test_main_warns_when_routing_profiles_flag_is_used( +def test_main_accepts_routing_profiles_flag( monkeypatch: pytest.MonkeyPatch, tmp_path, capsys: pytest.CaptureFixture[str], @@ -578,13 +530,10 @@ def _fake_serve( cli.main() - stderr = capsys.readouterr().err - assert "warning: --routing-profiles is deprecated." in stderr - assert "switchyard serve --config PATH" in stderr - assert "removed in a future release" in stderr + assert capsys.readouterr().err == "" -def test_serve_warns_when_saved_route_bundle_is_used( +def test_serve_accepts_saved_route_bundle( monkeypatch: pytest.MonkeyPatch, tmp_path, capsys: pytest.CaptureFixture[str], @@ -607,10 +556,7 @@ def _fake_serve( cli._cmd_serve(args) - stderr = capsys.readouterr().err - assert "warning: saved routing-profile bundle is deprecated." in stderr - assert "switchyard serve --config PATH" in stderr - assert "Clear the saved bundle" in stderr + assert capsys.readouterr().err == "" def test_serve_subcommand_enables_intake_from_cli_args( diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py deleted file mode 100644 index 299cb4ef..00000000 --- a/tests/test_serve_profile_config.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for serving a v2 profile config via `serve --config`.""" - -import argparse -import json -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any - -import pytest -from fastapi.testclient import TestClient - -from switchyard.cli.switchyard_cli import _cmd_serve_profile_config, _profile_config_route_table -from switchyard.server.switchyard_app import build_switchyard_app - -_PYTHON_CONFIG = """ -targets: - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:9/weak/v1 - api_key: test-key - -profiles: - smart: - type: header-routing - strong: weak - weak: weak -""" - - -def _mixed_config(base_url: str) -> str: - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/strong/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/weak/v1" - api_key: test-key - -profiles: - direct: - type: passthrough - target: weak - smart: - type: header-routing - strong: strong - weak: weak -""" - - -class _MockOpenAIServer(ThreadingHTTPServer): - calls: list[dict[str, Any]] - - def __init__(self) -> None: - super().__init__(("127.0.0.1", 0), _MockOpenAIHandler) - self.calls = [] - - @property - def base_url(self) -> str: - host, port = self.server_address - return f"http://{host}:{port}" - - -class _MockOpenAIHandler(BaseHTTPRequestHandler): - server: _MockOpenAIServer - - def do_POST(self) -> None: - content_length = int(self.headers.get("content-length", "0")) - raw_body = self.rfile.read(content_length) - body = json.loads(raw_body.decode("utf-8")) if raw_body else {} - self.server.calls.append({"path": self.path, "body": body}) - response = { - "id": "chatcmpl-serve-profile-config", - "object": "chat.completion", - "model": body.get("model"), - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop", - } - ], - } - payload = json.dumps(response).encode("utf-8") - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - -@pytest.fixture -def mock_openai_server() -> _MockOpenAIServer: - try: - server = _MockOpenAIServer() - except PermissionError as exc: - pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}") - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield server - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - -def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path: - path = tmp_path / name - path.write_text(text, encoding="utf-8") - return path - - -def test_serve_config_uses_fastapi_profile_table( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _PYTHON_CONFIG) - args = _serve_args(path) - captured: dict[str, Any] = {} - - def capture_build_and_serve( - _args: argparse.Namespace, - table: Any, - inbound_default: str = "openai", - disable_backend_streaming: bool = False, - extra_endpoints: list[Any] | None = None, - ) -> None: - captured["models"] = table.registered_models() - captured["inbound_default"] = inbound_default - captured["disable_backend_streaming"] = disable_backend_streaming - captured["extra_endpoints"] = extra_endpoints - - monkeypatch.setattr( - "switchyard.cli.switchyard_cli.build_and_serve", - capture_build_and_serve, - ) - - _cmd_serve_profile_config(args) - - assert captured["models"] == ["smart", "weak", "provider/weak"] - assert captured["inbound_default"] == "both" - - -def test_profile_config_route_table_serves_mixed_profiles( - mock_openai_server: _MockOpenAIServer, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _mixed_config(mock_openai_server.base_url)) - table = _profile_config_route_table(str(path)) - - assert table.registered_models() == [ - "direct", - "smart", - "strong", - "provider/strong", - "weak", - "provider/weak", - ] - - with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: - models_response = client.get("/v1/models") - assert models_response.status_code == 200 - assert models_response.json()["model_pool"] == [ - "direct", - "smart", - "strong", - "provider/strong", - "weak", - "provider/weak", - ] - - responses = { - path: client.post( - path, - headers={"x-switchyard-tier": "strong"}, - json=body, - ) - for path, body in ( - ( - "/v1/chat/completions", - { - "model": "smart", - "messages": [{"role": "user", "content": "hi"}], - }, - ), - ( - "/v1/messages", - { - "model": "smart", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - }, - ), - ("/v1/responses", {"model": "smart", "input": "hi"}), - ) - } - - assert [response.status_code for response in responses.values()] == [200, 200, 200] - assert [call["path"] for call in mock_openai_server.calls] == [ - "/strong/v1/chat/completions", - "/strong/v1/chat/completions", - "/strong/v1/chat/completions", - ] - assert responses["/v1/chat/completions"].json()["model"] == "provider/strong" - assert responses["/v1/messages"].json()["content"][0]["text"] == "ok" - assert ( - responses["/v1/responses"].json()["output"][0]["content"][0]["text"] - == "ok" - ) - - -def _serve_args(path: Path) -> argparse.Namespace: - return argparse.Namespace( - config=str(path), - routing_profiles=None, - inbound=None, - reload=False, - workers=1, - intake_enabled=False, - intake_base_url=None, - intake_workspace=None, - intake_api_key=None, - intake_target_url=None, - host="127.0.0.1", - port=4000, - ) diff --git a/tests/test_subagent_routing.py b/tests/test_subagent_routing.py deleted file mode 100644 index 310f9817..00000000 --- a/tests/test_subagent_routing.py +++ /dev/null @@ -1,319 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for envelope-level sub-agent routing (``subagent_target``). - -A profile config may name a ``subagent_target`` in its common envelope; the -loader then wraps the built profile so recognized sub-agent requests run -through a passthrough to that target while all other traffic keeps the -profile's own routing. -""" - -import json -import threading -from collections.abc import Iterator -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any - -import pytest - -from switchyard import load_profiles -from switchyard.lib.profiles import ProfileConfigError -from switchyard.lib.profiles.subagent_override import SubagentOverrideProfile -from switchyard_rust.core import ChatRequest, SwitchyardConfigError -from switchyard_rust.profiles import ( - ProfileInput, - ProfileRequestMetadata, - is_subagent_request, -) - -SUBAGENT_HEADERS = { - "x-claude-code-session-id": "root-session", - "x-claude-code-agent-id": "worker-1", -} - - -class _MockOpenAIServer(ThreadingHTTPServer): - calls: list[dict[str, Any]] - - def __init__(self) -> None: - super().__init__(("127.0.0.1", 0), _MockOpenAIHandler) - self.calls = [] - - @property - def base_url(self) -> str: - host, port = self.server_address - return f"http://{host}:{port}" - - -class _MockOpenAIHandler(BaseHTTPRequestHandler): - server: _MockOpenAIServer - - def do_POST(self) -> None: - content_length = int(self.headers.get("content-length", "0")) - raw_body = self.rfile.read(content_length) - body = json.loads(raw_body.decode("utf-8")) if raw_body else {} - self.server.calls.append({"path": self.path, "body": body}) - response = { - "id": "chatcmpl-subagent", - "object": "chat.completion", - "model": body.get("model"), - "mock_path": self.path, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, - } - payload = json.dumps(response).encode("utf-8") - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message(self, _format: str, *_args: object) -> None: - return - - -@pytest.fixture -def mock_openai_server() -> Iterator[_MockOpenAIServer]: - try: - server = _MockOpenAIServer() - except PermissionError as exc: - pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}") - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield server - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - -def _config(base_url: str) -> str: - """Rust and Python profiles that both name a ``subagent_target``.""" - return f""" -targets: - strong: - model: provider/strong - format: openai - base_url: "{base_url}/strong/v1" - api_key: test-key - worker: - model: provider/worker - format: openai - base_url: "{base_url}/worker/v1" - api_key: test-key - -profiles: - direct: - type: passthrough - target: strong - subagent_target: worker - smart: - type: header-routing - strong: strong - weak: strong - subagent_target: worker - plain: - type: passthrough - target: strong -""" - - -def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path: - path = tmp_path / name - path.write_text(text, encoding="utf-8") - return path - - -def _request(model: str = "client/x") -> ChatRequest: - return ChatRequest.openai_chat( - {"model": model, "messages": [{"role": "user", "content": "hello"}]} - ) - - -def _subagent_input() -> ProfileInput: - return ProfileInput( - _request(), - ProfileRequestMetadata(headers=dict(SUBAGENT_HEADERS)), - ) - - -# --- Detection ------------------------------------------------------------ - - -@pytest.mark.parametrize( - ("headers", "expected"), - [ - # No signal at all. - ({}, False), - # Claude Code lineage: any non-empty agent id marks a child agent. - (SUBAGENT_HEADERS, True), - # Agent id alone (no session header) is still a child agent. - ({"x-claude-code-agent-id": "child-1"}, True), - # Session alone — root agent, no agent-id sent. - ({"x-claude-code-session-id": "s"}, False), - # Codex delegated-work kinds route as sub-agent work. - ({"x-openai-subagent": "review"}, True), - ({"x-openai-subagent": "collab_spawn"}, True), - # Codex harness maintenance and unknown kinds stay on normal routing. - ({"x-openai-subagent": "compact"}, False), - ({"x-openai-subagent": "memory_consolidation"}, False), - ({"x-openai-subagent": "brand_new_kind"}, False), - # The explicit Switchyard header decides the lineage fact, in both - # directions; the work-kind policy still applies on top of it. - ({"x-switchyard-is-subagent": "true"}, True), - ({"x-switchyard-is-subagent": "false", "x-openai-subagent": "review"}, False), - ({"x-switchyard-is-subagent": "true", "x-openai-subagent": "compact"}, False), - # Header names are case-insensitive and values are trimmed. - ({"X-OpenAI-Subagent": " review "}, True), - # Bug #2 fix: operator label x-switchyard-agent-kind must not suppress - # routing signals from the harness (x-openai-subagent, x-switchyard-is-subagent). - ({"x-openai-subagent": "review", "x-switchyard-agent-kind": "researcher"}, True), - ({"x-switchyard-is-subagent": "true", "x-switchyard-agent-kind": "researcher"}, True), - # Bug #1 fix: correlation-only headers are not routing signals. - ({"x-switchyard-parent-agent-id": "parent"}, False), - ({"x-nemo-relay-subagent-id": "relay-child"}, False), - ({"x-dynamo-parent-session-id": "dynamo-parent"}, False), - ({"x-session-id": "s", "x-parent-session-id": "p"}, False), - ], -) -def test_is_subagent_request_detection(headers: dict[str, str], expected: bool) -> None: - assert is_subagent_request(headers) is expected - - -# --- Loader wiring ---------------------------------------------------------- - - -def test_loader_wraps_profiles_with_subagent_target(tmp_path: Path) -> None: - path = _write(tmp_path, _config("http://127.0.0.1:9")) - profiles = load_profiles(path) - # Rust profiles (direct / passthrough) handle subagent routing at the Rust level; - # the Python loader must not double-wrap them with SubagentOverrideProfile. - assert not isinstance(profiles["direct"], SubagentOverrideProfile) - # Python profiles (smart / header-routing) still receive the Python wrapper. - assert isinstance(profiles["smart"], SubagentOverrideProfile) - assert not isinstance(profiles["plain"], SubagentOverrideProfile) - - -async def test_rust_profile_routes_subagent_requests_to_target( - mock_openai_server: _MockOpenAIServer, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _config(mock_openai_server.base_url)) - profiles = load_profiles(path) - - normal = await profiles["direct"].run(ProfileInput(_request())) - assert normal.body["model"] == "provider/strong" - assert normal.body["mock_path"] == "/strong/v1/chat/completions" - - subagent = await profiles["direct"].run(_subagent_input()) - assert subagent.body["model"] == "provider/worker" - assert subagent.body["mock_path"] == "/worker/v1/chat/completions" - - -async def test_python_profile_routes_subagent_requests_to_target( - mock_openai_server: _MockOpenAIServer, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _config(mock_openai_server.base_url)) - profiles = load_profiles(path) - - normal = await profiles["smart"].run(ProfileInput(_request())) - assert normal.body["model"] == "provider/strong" - - subagent = await profiles["smart"].run(_subagent_input()) - assert subagent.body["model"] == "provider/worker" - assert subagent.body["mock_path"] == "/worker/v1/chat/completions" - - -async def test_maintenance_kinds_keep_normal_routing( - mock_openai_server: _MockOpenAIServer, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _config(mock_openai_server.base_url)) - profiles = load_profiles(path) - - metadata = ProfileRequestMetadata(headers={"x-openai-subagent": "compact"}) - response = await profiles["direct"].run(ProfileInput(_request(), metadata)) - assert response.body["model"] == "provider/strong" - - -def test_rust_profile_unknown_subagent_target_is_rejected(tmp_path: Path) -> None: - config = """ -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:9/v1 - api_key: test-key - -profiles: - direct: - type: passthrough - target: strong - subagent_target: ghost -""" - path = _write(tmp_path, config) - with pytest.raises(SwitchyardConfigError, match="unknown target ghost"): - load_profiles(path) - - -def test_python_profile_unknown_subagent_target_is_rejected(tmp_path: Path) -> None: - config = """ -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:9/v1 - api_key: test-key - -profiles: - smart: - type: header-routing - strong: strong - weak: strong - subagent_target: ghost -""" - path = _write(tmp_path, config) - with pytest.raises(ProfileConfigError, match="unknown target 'ghost'"): - load_profiles(path) - - -# --- Wrapper behavior ------------------------------------------------------- - - -class _StubRunner: - def __init__(self, label: str) -> None: - self.label = label - self.calls = 0 - - async def run(self, input: ProfileInput) -> Any: - self.calls += 1 - return self.label - - -def test_iter_components_spans_both_branches() -> None: - inner = _StubRunner("inner") - override = _StubRunner("override") - wrapper = SubagentOverrideProfile(inner, override) - assert wrapper.iter_components() == [inner, override] - - -async def test_override_failure_is_not_rerouted_to_the_wrapped_profile() -> None: - class _FailingRunner: - async def run(self, input: ProfileInput) -> Any: - raise RuntimeError("worker target unavailable") - - inner = _StubRunner("inner") - wrapper = SubagentOverrideProfile(inner, _FailingRunner()) - with pytest.raises(RuntimeError, match="worker target unavailable"): - await wrapper.run(_subagent_input()) - assert inner.calls == 0 diff --git a/tests/test_switchyard_rust_profile_bindings.py b/tests/test_switchyard_rust_profile_bindings.py deleted file mode 100644 index 1ff8aada..00000000 --- a/tests/test_switchyard_rust_profile_bindings.py +++ /dev/null @@ -1,429 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for components-v2 profile config and runtime bindings.""" - -import json -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Any - -import pytest - -from switchyard_rust.components import LlmTarget -from switchyard_rust.core import ChatRequest, ChatRequestType, SwitchyardConfigError -from switchyard_rust.profiles import ( - Profile, - ProfileConfigDocument, - ProfileConfigPlan, - ProfileInput, - ProfileRequestMetadata, - load_profile_config, - parse_profile_config_path, - parse_profile_config_str, -) - - -class _MockOpenAIServer(ThreadingHTTPServer): - calls: list[dict[str, Any]] - - def __init__(self) -> None: - super().__init__(("127.0.0.1", 0), _MockOpenAIHandler) - self.calls = [] - - @property - def base_url(self) -> str: - host, port = self.server_address - return f"http://{host}:{port}" - - -class _MockOpenAIHandler(BaseHTTPRequestHandler): - server: _MockOpenAIServer - - def do_POST(self) -> None: - content_length = int(self.headers.get("content-length", "0")) - raw_body = self.rfile.read(content_length) - body = json.loads(raw_body.decode("utf-8")) if raw_body else {} - self.server.calls.append({"path": self.path, "body": body}) - - message: dict[str, Any] = {"role": "assistant", "content": "ok"} - finish_reason = "stop" - if body.get("tools"): - tool_name = body["tool_choice"]["function"]["name"] - message = { - "role": "assistant", - "tool_calls": [ - { - "id": "call_route", - "type": "function", - "function": { - "name": tool_name, - "arguments": json.dumps( - { - "recommended_tier": "medium", - "confidence": 0.9, - "abstain": False, - "turn_type": "exploration", - "code_modification_scope": "none", - "tool_call_count_estimate": 0, - "requires_codebase_context": False, - } - ), - }, - } - ], - } - finish_reason = "tool_calls" - elif body.get("response_format", {}).get("type") == "json_schema": - schema_name = body["response_format"]["json_schema"]["name"] - if schema_name == "StageRouterTierDecision": - message = {"role": "assistant", "content": json.dumps({"tier": "efficient"})} - elif ( - body.get("response_format", {}).get("type") == "json_object" - and body.get("model") == "provider/classifier" - ): - message = {"role": "assistant", "content": json.dumps({"tier": "efficient"})} - - response = { - "id": "chatcmpl-profile-bindings", - "object": "chat.completion", - "model": body.get("model"), - "mock_path": self.path, - "choices": [ - { - "index": 0, - "message": message, - "finish_reason": finish_reason, - } - ], - "usage": { - "prompt_tokens": 2, - "completion_tokens": 1, - "total_tokens": 3, - }, - } - payload = json.dumps(response).encode("utf-8") - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message( - self, - _format: str, - _requestline: str | None = None, - _code: str | None = None, - _size: str | None = None, - ) -> None: - return - - -@pytest.fixture -def mock_openai_server() -> _MockOpenAIServer: - try: - server = _MockOpenAIServer() - except PermissionError as exc: - pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}") - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield server - finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() - - -def _profile_config(base_url: str) -> str: - return f""" -targets: - direct: - model: provider/direct - format: openai - base_url: "{base_url}/direct/v1" - api_key: test-key - strong: - model: provider/strong - format: openai - base_url: "{base_url}/strong/v1" - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: "{base_url}/weak/v1" - api_key: test-key - latency: - model: provider/latency - format: openai - base_url: "{base_url}/latency/v1" - api_key: test-key - classifier: - model: provider/classifier - format: openai - base_url: "{base_url}/classifier/v1" - api_key: test-key - -profiles: - direct: - type: passthrough - target: direct - random: - type: random-routing - strong: strong - weak: weak - strong_probability: 1.0 - rng_seed: 7 - latency: - type: latency-service - latency_service_url: "http://latency.invalid" - targets: [latency] - max_retries: 0 - llm: - type: llm-routing - strong: strong - weak: weak - classifier: classifier - profile_name: coding_agent - stage_router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.7 - classifier: - model: provider/classifier - api_key: test-key - base_url: "{base_url}/classifier/v1" - bench: - type: noop -""" - - -def _plan(base_url: str) -> ProfileConfigPlan: - document = parse_profile_config_str(_profile_config(base_url)) - assert isinstance(document, ProfileConfigDocument) - return document.resolve() - - -def _offline_plan() -> ProfileConfigPlan: - return _plan("http://127.0.0.1:9") - - -def _request(model: str) -> ChatRequest: - return ChatRequest.openai_chat( - { - "model": model, - "messages": [{"role": "user", "content": "hello"}], - } - ) - - -def test_profile_config_plan_is_inspectable() -> None: - document = parse_profile_config_str(_profile_config("http://127.0.0.1:9")) - - assert document.profile_ids() == [ - "bench", - "direct", - "latency", - "llm", - "random", - "stage_router", - ] - assert document.profile_type("direct") == "passthrough" - assert document.profile_body("random") == { - "strong": "strong", - "weak": "weak", - "strong_probability": 1.0, - "rng_seed": 7, - } - rust_document = document.without_profiles(["random"]) - assert rust_document.profile_ids() == ["bench", "direct", "latency", "llm", "stage_router"] - - plan = document.resolve() - - assert isinstance(plan, ProfileConfigPlan) - assert plan.profile_ids() == [ - "bench", - "direct", - "latency", - "llm", - "random", - "stage_router", - ] - assert plan.target_ids() == ["classifier", "direct", "latency", "strong", "weak"] - assert plan.profile_type("direct") == "passthrough" - assert plan.profile_type("random") == "random-routing" - assert plan.profile_type("latency") == "latency-service" - assert plan.profile_type("llm") == "llm-routing" - assert plan.profile_type("stage_router") == "stage_router" - assert plan.profile_type("bench") == "noop" - assert plan.profile_type("missing") is None - - target = plan.target("direct") - assert isinstance(target, LlmTarget) - assert target.id == "direct" - assert target.model == "provider/direct" - assert target.base_url == "http://127.0.0.1:9/direct/v1" - assert plan.target("missing") is None - - profiles = plan.build_profiles() - assert sorted(profiles) == [ - "bench", - "direct", - "latency", - "llm", - "random", - "stage_router", - ] - assert all(isinstance(profile, Profile) for profile in profiles.values()) - assert profiles["direct"].profile_id == "direct" - - -def test_profile_config_can_be_loaded_from_path( - tmp_path: Path, -) -> None: - path = tmp_path / "profiles.yaml" - path.write_text(_profile_config("http://127.0.0.1:9"), encoding="utf-8") - - document = parse_profile_config_path(path) - assert isinstance(document.resolve(), ProfileConfigPlan) - - plan = load_profile_config(path) - assert plan.profile_ids() == [ - "bench", - "direct", - "latency", - "llm", - "random", - "stage_router", - ] - - -async def test_noop_profile_returns_local_response() -> None: - profile = _offline_plan().build_profile("bench") - - assert isinstance(profile, Profile) - assert profile.profile_id == "bench" - response = await profile.run(_request("client/noop")) - - assert response.body["id"] == "switchyard-noop" - assert response.body["model"] == "client/noop" - assert response.body["choices"][0]["message"]["content"] == "ok" - - -async def test_native_profiles_run_against_local_openai_mock( - mock_openai_server: _MockOpenAIServer, -) -> None: - plan = _plan(mock_openai_server.base_url) - - direct_response = await plan.build_profile("direct").run(_request("client/direct")) - random_response = await plan.build_profile("random").run(_request("client/random")) - latency_response = await plan.build_profile("latency").run(_request("client/latency")) - llm_response = await plan.build_profile("llm").run(_request("client/llm-routing")) - stage_router_response = await plan.build_profile("stage_router").run(_request("client/stage_router")) - - assert direct_response.body["model"] == "provider/direct" - assert direct_response.body["mock_path"] == "/direct/v1/chat/completions" - assert random_response.body["model"] == "provider/strong" - assert random_response.body["mock_path"] == "/strong/v1/chat/completions" - assert latency_response.body["model"] == "provider/latency" - assert latency_response.body["mock_path"] == "/latency/v1/chat/completions" - assert llm_response.body["model"] == "provider/weak" - assert llm_response.body["mock_path"] == "/weak/v1/chat/completions" - assert stage_router_response.body["model"] == "provider/weak" - assert stage_router_response.body["mock_path"] == "/weak/v1/chat/completions" - assert [call["body"]["model"] for call in mock_openai_server.calls] == [ - "provider/direct", - "provider/strong", - "provider/latency", - "provider/classifier", - "provider/weak", - "provider/classifier", - "provider/weak", - ] - llm_classifier_body = mock_openai_server.calls[3]["body"] - assert llm_classifier_body["tools"][0]["function"]["strict"] is True - assert llm_classifier_body["tool_choice"]["function"]["name"] == "select_route" - assert "response_format" not in llm_classifier_body - stage_router_classifier_body = mock_openai_server.calls[5]["body"] - assert stage_router_classifier_body["response_format"]["type"] == "json_object" - - -def test_profile_binding_errors_map_to_switchyard_exceptions() -> None: - with pytest.raises(SwitchyardConfigError, match="unknown field.*routes"): - parse_profile_config_str("routes: {}\n") - - unknown_target = """ -targets: {} -profiles: - bad: - type: passthrough - target: missing -""" - with pytest.raises(SwitchyardConfigError, match="profile bad:.*unknown target missing"): - parse_profile_config_str(unknown_target).resolve() - - with pytest.raises(SwitchyardConfigError, match="unknown profile missing"): - parse_profile_config_str("profiles:\n bench:\n type: noop\n").resolve().build_profile( - "missing" - ) - - -def test_profile_request_metadata_normalizes_headers() -> None: - metadata = ProfileRequestMetadata.from_headers( - { - "X-Request-ID": "req-123", - "X-Switchyard-Trace": ["trace-a", "trace-b"], - }, - inbound_format=ChatRequestType.OPENAI_CHAT, - ) - assert metadata.request_id == "req-123" - assert metadata.inbound_format == ChatRequestType.OPENAI_CHAT - assert metadata.headers == { - "x-request-id": ["req-123"], - "x-switchyard-trace": ["trace-a", "trace-b"], - } - - -def test_profile_input_binding_wraps_request_and_metadata() -> None: - metadata = ProfileRequestMetadata( - request_id="req-profile-input", - inbound_format=ChatRequestType.OPENAI_CHAT, - headers={"X-Switchyard-Trace": "trace-profile-input"}, - ) - - input = ProfileInput(_request("client/profile-input"), metadata=metadata) - - assert input.request.model == "client/profile-input" - assert input.metadata.request_id == "req-profile-input" - assert input.metadata.inbound_format == ChatRequestType.OPENAI_CHAT - assert input.metadata.headers == {"x-switchyard-trace": ["trace-profile-input"]} - - -def test_concrete_profile_trios_are_not_reexported_from_rust_profiles() -> None: - import switchyard_rust.profiles as profiles - - for name in ( - "PassthroughProfileConfig", - "PassthroughProfile", - "PassthroughProcessedRequest", - "RandomRoutingProfileConfig", - "RandomRoutingProfile", - "RandomRoutingProcessedRequest", - "LatencyServiceProfileConfig", - "LatencyServiceProfile", - "LatencyServiceProcessedRequest", - "LlmRoutingProfileConfig", - "LlmRoutingProfile", - "LlmRoutingProcessedRequest", - "StageRouterProfileConfig", - "StageRouterProfile", - "StageRouterProcessedRequest", - "NoopProfileConfig", - "NoopProfile", - "NoopProcessedRequest", - ): - assert not hasattr(profiles, name)