From 475c865125922082bfc79788021b96e68bf7b71e Mon Sep 17 00:00:00 2001 From: SiddarthAA Date: Fri, 28 Aug 2026 13:00:49 +0530 Subject: [PATCH 1/4] docs(sdk): clarify evaluator v2 boundary --- sdk/python/CHANGELOG.md | 3 +++ sdk/python/README.md | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 57aa24a9..5e763997 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,6 +16,9 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. +- Document that the retired inbound evaluator package is not part of this SDK + and reserve evaluator authoring for the forthcoming outbound-only v2 runtime. + ## 0.0.1b1 — 2026-08-24 The first release under this name. Everything below describes the package as it diff --git a/sdk/python/README.md b/sdk/python/README.md index f974587d..e846b623 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -10,6 +10,15 @@ the platform. - **Dependencies:** none. Standard library only, so installing it constrains nothing else in your environment. +## Evaluator v2 status + +This package currently provides tracing and event emission only. The legacy +inbound `agenteye-evaluator` package has been retired; do not build new evaluator +services against its server-push HTTP contract. A customer-hosted, outbound-only +worker runtime will be added under the lazy `failproofai_sdk.evaluator` namespace +as part of Evaluator v2. Until that API ships, no evaluator module is included in +the distribution. + ## Installation ```bash From d27fee0f40c983c4309c6bf47b8ddbc7bb2a03eb Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 16:04:04 +0530 Subject: [PATCH 2/4] feat(sdk): add evaluator v2 worker runtime --- sdk/python/CHANGELOG.md | 5 +- sdk/python/README.md | 11 +- sdk/python/examples/evaluator_worker.py | 111 +++ .../failproofai_sdk/evaluator/__init__.py | 85 +++ .../failproofai_sdk/evaluator/__main__.py | 49 ++ .../failproofai_sdk/evaluator/authoring.py | 377 ++++++++++ .../failproofai_sdk/evaluator/client.py | 268 +++++++ .../failproofai_sdk/evaluator/protocol.py | 659 ++++++++++++++++ .../failproofai_sdk/evaluator/runtime.py | 563 ++++++++++++++ .../tests/fixtures/evaluator_v2/README.md | 28 + .../tests/fixtures/evaluator_v2/contract.json | 225 ++++++ sdk/python/tests/test_evaluator_authoring.py | 124 ++++ sdk/python/tests/test_evaluator_client.py | 221 ++++++ sdk/python/tests/test_evaluator_example.py | 35 + sdk/python/tests/test_evaluator_http_e2e.py | 631 ++++++++++++++++ sdk/python/tests/test_evaluator_main.py | 47 ++ sdk/python/tests/test_evaluator_protocol.py | 223 ++++++ sdk/python/tests/test_evaluator_runtime.py | 701 ++++++++++++++++++ sdk/python/tests/test_zero_dependencies.py | 17 + 19 files changed, 4372 insertions(+), 8 deletions(-) create mode 100644 sdk/python/examples/evaluator_worker.py create mode 100644 sdk/python/failproofai_sdk/evaluator/__init__.py create mode 100644 sdk/python/failproofai_sdk/evaluator/__main__.py create mode 100644 sdk/python/failproofai_sdk/evaluator/authoring.py create mode 100644 sdk/python/failproofai_sdk/evaluator/client.py create mode 100644 sdk/python/failproofai_sdk/evaluator/protocol.py create mode 100644 sdk/python/failproofai_sdk/evaluator/runtime.py create mode 100644 sdk/python/tests/fixtures/evaluator_v2/README.md create mode 100644 sdk/python/tests/fixtures/evaluator_v2/contract.json create mode 100644 sdk/python/tests/test_evaluator_authoring.py create mode 100644 sdk/python/tests/test_evaluator_client.py create mode 100644 sdk/python/tests/test_evaluator_example.py create mode 100644 sdk/python/tests/test_evaluator_http_e2e.py create mode 100644 sdk/python/tests/test_evaluator_main.py create mode 100644 sdk/python/tests/test_evaluator_protocol.py create mode 100644 sdk/python/tests/test_evaluator_runtime.py diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 5e763997..74d6b5b3 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,8 +16,9 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. -- Document that the retired inbound evaluator package is not part of this SDK - and reserve evaluator authoring for the forthcoming outbound-only v2 runtime. +- Retire the old inbound evaluator boundary and add evaluator authoring plus the + outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator` + namespace. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/README.md b/sdk/python/README.md index e846b623..3cb8adaf 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -12,12 +12,11 @@ the platform. ## Evaluator v2 status -This package currently provides tracing and event emission only. The legacy -inbound `agenteye-evaluator` package has been retired; do not build new evaluator -services against its server-push HTTP contract. A customer-hosted, outbound-only -worker runtime will be added under the lazy `failproofai_sdk.evaluator` namespace -as part of Evaluator v2. Until that API ships, no evaluator module is included in -the distribution. +The legacy inbound `agenteye-evaluator` package has been retired; do not build new +evaluator services against its server-push HTTP contract. Evaluator v2 authoring +and its customer-hosted, outbound-only worker runtime live under the lazy +`failproofai_sdk.evaluator` namespace. Importing the top-level tracing SDK does not +import or start the evaluator runtime. ## Installation diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py new file mode 100644 index 00000000..a0cea74b --- /dev/null +++ b/sdk/python/examples/evaluator_worker.py @@ -0,0 +1,111 @@ +"""Customer evaluator with deterministic and optional async judge checks.""" + +from __future__ import annotations + +import asyncio +import json +import os +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator import ( + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, +) + +app = Evaluator(name="customer-production", version="2026.08.1") + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +@app.eval( + "tool_efficiency", + version="1.0.0", + labels=["tools", "deterministic"], + when=lambda session: ConditionResult( + session.count("tool_use") > 0, "no_tool_calls" + ), +) +def tool_efficiency(session): + calls = session.events_of_type("tool_use") + distinct = { + event.payload.get("tool_name") + for event in calls + if event.payload.get("tool_name") + } + value = len(distinct) / len(calls) + return EvalResult( + score=Score(value, passed=value >= 0.7), + metrics={ + "tool_call_count": Metric(len(calls), unit="events"), + "distinct_tool_count": Metric(len(distinct), unit="tools"), + }, + reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls", + ) + + +def _judge_configured(session): + configured = bool(os.environ.get("EXAMPLE_JUDGE_URL")) + return ConditionResult(configured, "judge_not_configured") + + +def _last_content(session, event_type): + events = session.events_of_type(event_type) + if not events: + return None + payload = events[-1].payload + fields = { + "human_input": ("response",), + "model_response": ("content",), + "agent_end": ("summary",), + }.get(event_type, ("content", "summary", "response")) + return next((payload.get(field) for field in fields if payload.get(field)), None) + + +def _call_judge(question, answer): + url = os.environ["EXAMPLE_JUDGE_URL"] + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + token = os.environ.get("EXAMPLE_JUDGE_TOKEN") + body = json.dumps({"question": question, "answer": answer}).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + request = Request(url, data=body, headers=headers, method="POST") + with build_opener(_RejectRedirects()).open(request, timeout=25) as response: # nosec B310 + result = json.loads(response.read(64 * 1024)) + return float(result["score"]), str( + result.get("reasoning") or "Judge returned no reasoning" + ) + + +@app.eval( + "answer_relevance", + version="judge-api-v1", + labels=["llm_judge", "relevance"], + when=_judge_configured, + timeout_seconds=30, +) +async def answer_relevance(session): + question = _last_content(session, "human_input") + answer = _last_content(session, "model_response") + if question is None or answer is None: + raise ValueError("answer relevance requires human input and model output") + value, reasoning = await asyncio.to_thread(_call_judge, question, answer) + value = min(max(value, 0.0), 1.0) + return EvalResult( + score=Score(value, passed=value >= 0.7), + reasoning=reasoning, + labels=("llm_judge", "relevance"), + ) + + +if __name__ == "__main__": + app.run_from_env() diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py new file mode 100644 index 00000000..dff0cad6 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -0,0 +1,85 @@ +"""Authoring and worker primitives for FailproofAI Evaluator v2. + +This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users +who only emit telemetry do not import evaluator networking or runtime code. +""" + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, + Metric, + Score, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + Assignment, + CatalogDefinition, + ClaimRequest, + ClaimResponse, + ErrorResponse, + EvalSelection, + EvaluatorKind, + HeartbeatRequest, + HeartbeatResponse, + HeartbeatRun, + PlannedRun, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + RemoteError, + ResultItem, + ResultKind, + ResultRequest, + ResultResponse, + SessionTranscript, + SkippedEval, + TerminalRunStatus, + TranscriptEvent, + UnsupportedProtocolVersion, +) +from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + +__all__ = [ + "Assertion", + "Assignment", + "CatalogDefinition", + "ClaimRequest", + "ClaimResponse", + "ConditionResult", + "ErrorResponse", + "EvalDefinition", + "EvalResult", + "EvalSelection", + "Evaluator", + "EvaluatorAPIError", + "EvaluatorClient", + "EvaluatorKind", + "HeartbeatRequest", + "HeartbeatResponse", + "HeartbeatRun", + "Metric", + "PlanRequest", + "PlanResponse", + "PlannedRun", + "ProtocolError", + "RegisterRequest", + "RegisterResponse", + "RemoteError", + "ResultItem", + "ResultKind", + "ResultRequest", + "ResultResponse", + "Score", + "SessionTranscript", + "SkippedEval", + "TerminalRunStatus", + "TranscriptEvent", + "UnsupportedProtocolVersion", + "WorkerConfig", + "WorkerRuntime", +] diff --git a/sdk/python/failproofai_sdk/evaluator/__main__.py b/sdk/python/failproofai_sdk/evaluator/__main__.py new file mode 100644 index 00000000..f7bff1c5 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__main__.py @@ -0,0 +1,49 @@ +"""Run an evaluator declared as ``module:attribute``.""" + +from __future__ import annotations + +import argparse +import importlib +import os +from collections.abc import Sequence + +from failproofai_sdk.evaluator.authoring import Evaluator + + +def load_evaluator(spec: str) -> Evaluator: + module_name, separator, attribute = spec.partition(":") + if not module_name: + raise ValueError("evaluator module must not be empty") + if not separator: + attribute = "app" + if not attribute: + raise ValueError("evaluator attribute must not be empty") + module = importlib.import_module(module_name) + try: + evaluator = getattr(module, attribute) + except AttributeError as error: + raise ValueError(f"{spec!r} does not define {attribute!r}") from error + if not isinstance(evaluator, Evaluator): + raise TypeError( + f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator" + ) + return evaluator + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator") + parser.add_argument( + "module", + nargs="?", + default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"), + help="Python module and optional attribute (for example my_evals:app)", + ) + args = parser.parse_args(argv) + if not args.module: + parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)") + load_evaluator(args.module).run_from_env() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/failproofai_sdk/evaluator/authoring.py b/sdk/python/failproofai_sdk/evaluator/authoring.py new file mode 100644 index 00000000..db1788d9 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/authoring.py @@ -0,0 +1,377 @@ +"""Evaluator definition registry and typed author results.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import re +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from failproofai_sdk.evaluator.protocol import ( + MAX_CATALOG_DEFINITIONS, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABEL_BYTES, + MAX_LABELS_PER_RESULT, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + CatalogDefinition, + ResultItem, + ResultKind, + SessionTranscript, +) + +_KEY = re.compile(r"^[a-z][a-z0-9_]*$") +EvalFunction = Callable[[SessionTranscript], "EvalResult | Awaitable[EvalResult]"] +ConditionFunction = Callable[ + [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]" +] +CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"] + + +def _bounded(value: str, *, field_name: str, maximum: int) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + if not value: + raise ValueError(f"{field_name} must not be empty") + size = len(value.encode("utf-8")) + if size > maximum: + raise ValueError(f"{field_name} is {size} bytes; maximum is {maximum}") + return value + + +def _finite(value: float, field_name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{field_name} must be a number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field_name} must be finite") + return result + + +def _labels(values: tuple[str, ...] | list[str]) -> tuple[str, ...]: + if len(values) > MAX_LABELS_PER_RESULT: + raise ValueError(f"at most {MAX_LABELS_PER_RESULT} labels are allowed") + normalized = [] + for label in values: + normalized.append(_bounded(label, field_name="label", maximum=MAX_LABEL_BYTES)) + if len(set(normalized)) != len(normalized): + raise ValueError("labels must be unique") + return tuple(sorted(normalized)) + + +@dataclass(frozen=True) +class Score: + value: float + passed: bool | None = None + unit: str = "ratio" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + value = _finite(self.value, "score value") + if not 0 <= value <= 1: + raise ValueError("score value must be between 0 and 1") + object.__setattr__(self, "value", value) + if self.passed is not None and not isinstance(self.passed, bool): + raise TypeError("score passed must be a boolean or None") + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Metric: + value: float + unit: str = "" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "value", _finite(self.value, "metric value")) + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Assertion: + passed: bool + description: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.passed, bool): + raise TypeError("assertion passed must be a boolean") + if self.description is not None: + _bounded( + self.description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class ConditionResult: + applicable: bool + reason_code: str = "condition_false" + + def __post_init__(self) -> None: + if not isinstance(self.applicable, bool): + raise TypeError("condition applicable must be a boolean") + _validate_key(self.reason_code, "condition reason code") + + +def _validate_result_text( + unit: str, display_value: str | None, description: str | None +) -> None: + if unit: + _bounded(unit, field_name="unit", maximum=MAX_UNIT_BYTES) + if display_value is not None: + _bounded( + display_value, + field_name="display value", + maximum=MAX_DISPLAY_VALUE_BYTES, + ) + if description is not None: + _bounded( + description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class EvalResult: + score: Score | None = None + metrics: Mapping[str, Metric | float] = field(default_factory=dict) + assertions: Mapping[str, Assertion | bool] = field(default_factory=dict) + reasoning: str | None = None + summary: str | None = None + labels: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.reasoning is not None: + _bounded( + self.reasoning, + field_name="reasoning", + maximum=MAX_REASONING_BYTES, + ) + if self.summary is not None: + _bounded(self.summary, field_name="summary", maximum=MAX_SUMMARY_BYTES) + object.__setattr__(self, "labels", _labels(list(self.labels))) + + def result_items(self, eval_key: str) -> tuple[ResultItem, ...]: + items: list[ResultItem] = [] + if self.score is not None: + items.append( + ResultItem( + result_key=eval_key, + result_kind=ResultKind.SCORE, + numeric_value=self.score.value, + bool_value=self.score.passed, + unit=self.score.unit, + display_value=self.score.display_value, + description=self.score.description, + reasoning=self.reasoning, + labels=self.labels, + ) + ) + for key, raw_metric in sorted(self.metrics.items()): + _validate_key(key, "metric key") + metric = ( + raw_metric if isinstance(raw_metric, Metric) else Metric(raw_metric) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.METRIC, + numeric_value=metric.value, + unit=metric.unit, + display_value=metric.display_value, + description=metric.description, + labels=self.labels, + ) + ) + for key, raw_assertion in sorted(self.assertions.items()): + _validate_key(key, "assertion key") + assertion = ( + raw_assertion + if isinstance(raw_assertion, Assertion) + else Assertion(raw_assertion) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.ASSERTION, + bool_value=assertion.passed, + description=assertion.description, + labels=self.labels, + ) + ) + if not items: + raise ValueError("an EvalResult must contain a score, metric, or assertion") + if len(items) > MAX_RESULTS_PER_RUN: + raise ValueError( + f"an EvalResult may contain at most {MAX_RESULTS_PER_RUN} results" + ) + keys = [item.result_key for item in items] + if len(keys) != len(set(keys)): + raise ValueError("result keys must be unique within one evaluation run") + return tuple(items) + + +def _validate_key(value: str, field_name: str = "eval_key") -> str: + _bounded(value, field_name=field_name, maximum=MAX_EVAL_KEY_BYTES) + if not _KEY.fullmatch(value): + raise ValueError(f"{field_name} must match {_KEY.pattern}") + return value + + +@dataclass(frozen=True) +class EvalDefinition: + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] + function: EvalFunction + condition: ConditionFunction | None + on_cancel: CancellationFunction | None + timeout_seconds: float | None + + def catalog_definition(self) -> CatalogDefinition: + return CatalogDefinition( + eval_key=self.eval_key, + display_name=self.display_name, + eval_version=self.eval_version, + result_kind=self.result_kind, + labels=self.labels, + ) + + +class Evaluator: + """A process-local collection of explicitly versioned evaluations.""" + + def __init__(self, *, name: str, version: str) -> None: + self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES) + self.version = _bounded( + version, field_name="version", maximum=MAX_VERSION_BYTES + ) + self._definitions: dict[str, EvalDefinition] = {} + + def eval( + self, + eval_key: str, + *, + version: str, + display_name: str | None = None, + result_kind: ResultKind | str = ResultKind.SCORE, + labels: tuple[str, ...] | list[str] = (), + when: ConditionFunction | None = None, + on_cancel: CancellationFunction | None = None, + timeout_seconds: float | None = None, + ) -> Callable[[EvalFunction], EvalFunction]: + key = _validate_key(eval_key) + eval_version = _bounded( + version, field_name="eval version", maximum=MAX_VERSION_BYTES + ) + display = _bounded( + display_name or eval_key.replace("_", " ").capitalize(), + field_name="display name", + maximum=MAX_DISPLAY_NAME_BYTES, + ) + kind = ResultKind(result_kind) + normalized_labels = _labels(list(labels)) + if timeout_seconds is not None: + timeout_seconds = _finite(timeout_seconds, "timeout_seconds") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + + def register(function: EvalFunction) -> EvalFunction: + if key in self._definitions: + raise ValueError(f"duplicate eval key: {key}") + if len(self._definitions) >= MAX_CATALOG_DEFINITIONS: + raise ValueError( + f"an evaluator may define at most {MAX_CATALOG_DEFINITIONS} evaluations" + ) + if not callable(function): + raise TypeError("evaluation must be callable") + if when is not None and not callable(when): + raise TypeError("when must be callable") + if on_cancel is not None and not callable(on_cancel): + raise TypeError("on_cancel must be callable") + self._definitions[key] = EvalDefinition( + eval_key=key, + display_name=display, + eval_version=eval_version, + result_kind=kind, + labels=normalized_labels, + function=function, + condition=when, + on_cancel=on_cancel, + timeout_seconds=timeout_seconds, + ) + return function + + return register + + @property + def definitions(self) -> tuple[EvalDefinition, ...]: + return tuple(self._definitions[key] for key in sorted(self._definitions)) + + def catalog(self) -> tuple[CatalogDefinition, ...]: + return tuple(definition.catalog_definition() for definition in self.definitions) + + @property + def catalog_revision(self) -> str: + payload = [item.to_wire() for item in self.catalog()] + canonical = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(canonical).hexdigest() + + def definition(self, eval_key: str) -> EvalDefinition: + try: + return self._definitions[eval_key] + except KeyError as error: + raise KeyError(f"unknown eval key: {eval_key}") from error + + def run_from_env(self) -> None: + """Run this evaluator until the process receives a stop request.""" + import asyncio + import signal + + from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + + async def run() -> None: + runtime = WorkerRuntime(self, WorkerConfig.from_env()) + loop = asyncio.get_running_loop() + for name in ("SIGINT", "SIGTERM"): + process_signal = getattr(signal, name, None) + if process_signal is None: + continue + try: + loop.add_signal_handler(process_signal, runtime.stop) + except (NotImplementedError, RuntimeError): + pass + await runtime.run_forever() + + asyncio.run(run()) + + @staticmethod + async def call( + function: EvalFunction | ConditionFunction, session: SessionTranscript + ) -> Any: + result = function(session) + if inspect.isawaitable(result): + return await result + return result diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py new file mode 100644 index 00000000..60fa32c2 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -0,0 +1,268 @@ +"""Standard-library HTTP client for the Evaluator v2 worker protocol.""" + +from __future__ import annotations + +import json +import random +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator.protocol import ( + CLAIM_PATH, + HEARTBEAT_PATH, + LEASE_GENERATION_HEADER, + MAX_TRANSCRIPT_BYTES, + PLAN_PATH, + REGISTER_PATH, + RESULT_PATH, + WORKER_ID_HEADER, + Assignment, + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + WireModel, +) + +_DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024 +_RETRYABLE_HTTP_STATUSES = frozenset({429, 502, 503, 504}) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def _open_without_redirects(request: Request, *, timeout: float): + return build_opener(_RejectRedirects()).open(request, timeout=timeout) + + +class EvaluatorAPIError(RuntimeError): + def __init__( + self, + *, + status: int | None, + code: str, + message: str, + retryable: bool, + request_id: str | None = None, + ) -> None: + super().__init__(f"{code}: {message}") + self.status = status + self.code = code + self.retryable = retryable + self.request_id = request_id + + +class EvaluatorClient: + """Direct server client; evaluator traffic never passes through the dashboard.""" + + def __init__( + self, + *, + base_url: str, + credential: str, + timeout_seconds: float = 30, + max_retries: int = 3, + opener: Callable[..., Any] | None = None, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + parsed = urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("base_url must be an absolute http(s) URL") + if not credential or not credential.strip(): + raise ValueError("credential must not be empty") + if any( + ord(character) < 32 or ord(character) == 127 for character in credential + ): + raise ValueError("credential must not contain control characters") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if max_retries < 0: + raise ValueError("max_retries must not be negative") + self._base_url = base_url.rstrip("/") + "/" + self._origin = (parsed.scheme, parsed.netloc) + self._credential = credential + self._timeout_seconds = timeout_seconds + self._max_retries = max_retries + self._opener = opener or _open_without_redirects + self._sleeper = sleeper + + def register(self, request: RegisterRequest) -> RegisterResponse: + return RegisterResponse.from_wire( + self._json("POST", REGISTER_PATH, request, retry=True) + ) + + def claim(self, request: ClaimRequest) -> ClaimResponse: + # A lost claim response may already have leased work. Do not hide a + # second claim behind transport retry; the runtime recalculates capacity. + return ClaimResponse.from_wire( + self._json("POST", CLAIM_PATH, request, retry=False) + ) + + def transcript( + self, assignment: Assignment, *, worker_id: str + ) -> SessionTranscript: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + return SessionTranscript.from_wire( + self._json( + "GET", + assignment.transcript_url, + None, + retry=True, + headers=headers, + response_limit=MAX_TRANSCRIPT_BYTES, + ) + ) + + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: + return PlanResponse.from_wire( + self._json( + "POST", + PLAN_PATH.format(assignment_id=assignment_id), + request, + retry=True, + ) + ) + + def heartbeat(self, request: HeartbeatRequest) -> HeartbeatResponse: + return HeartbeatResponse.from_wire( + self._json("POST", HEARTBEAT_PATH, request, retry=True) + ) + + def submit_result(self, run_id: str, request: ResultRequest) -> ResultResponse: + return ResultResponse.from_wire( + self._json( + "POST", + RESULT_PATH.format(evaluation_run_id=run_id), + request, + retry=True, + ) + ) + + def _url(self, path: str) -> str: + url = urljoin(self._base_url, path) + parsed = urlsplit(url) + if (parsed.scheme, parsed.netloc) != self._origin: + raise EvaluatorAPIError( + status=None, + code="invalid_transcript_url", + message="server supplied a URL outside the configured API origin", + retryable=False, + ) + return url + + def _json( + self, + method: str, + path: str, + body: WireModel | None, + *, + retry: bool, + headers: Mapping[str, str] | None = None, + response_limit: int = _DEFAULT_RESPONSE_LIMIT, + ) -> dict[str, Any]: + encoded = None + request_headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self._credential}", + "User-Agent": "failproofai-sdk-evaluator/2", + } + if body is not None: + encoded = json.dumps( + body.to_wire(), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + + attempts = self._max_retries + 1 if retry else 1 + for attempt in range(attempts): + request = Request( + self._url(path), data=encoded, headers=request_headers, method=method + ) + try: + with self._opener(request, timeout=self._timeout_seconds) as response: + return self._decode( + response.read(response_limit + 1), response_limit + ) + except HTTPError as error: + api_error = self._http_error(error, response_limit) + if attempt + 1 == attempts or not api_error.retryable: + raise api_error from error + except (URLError, TimeoutError, OSError) as error: + if attempt + 1 == attempts: + raise EvaluatorAPIError( + status=None, + code="transport_error", + message=str(error), + retryable=True, + ) from error + # Jitter is scheduling noise, not a security decision. + self._sleeper(random.uniform(0, min(0.25 * (2**attempt), 2.0))) # nosec B311 + raise AssertionError("retry loop exhausted without returning or raising") + + @staticmethod + def _decode(raw: bytes, limit: int) -> dict[str, Any]: + if len(raw) > limit: + raise EvaluatorAPIError( + status=None, + code="response_too_large", + message=f"server response exceeds {limit} bytes", + retryable=False, + ) + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response was not valid JSON", + retryable=False, + ) from error + if not isinstance(value, dict): + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response must be a JSON object", + retryable=False, + ) + return value + + @classmethod + def _http_error(cls, error: HTTPError, limit: int) -> EvaluatorAPIError: + raw = error.read(limit + 1) + try: + response = ErrorResponse.from_wire(cls._decode(raw, limit)) + except (ValueError, EvaluatorAPIError): + return EvaluatorAPIError( + status=error.code, + code="http_error", + message=f"server returned HTTP {error.code}", + retryable=error.code in _RETRYABLE_HTTP_STATUSES, + ) + return EvaluatorAPIError( + status=error.code, + code=response.error.code, + message=response.error.message, + retryable=response.error.retryable, + request_id=response.error.request_id, + ) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py new file mode 100644 index 00000000..09adf8be --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -0,0 +1,659 @@ +"""Dependency-free wire models for the outbound Evaluator v2 protocol.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any, TypeVar + +PROTOCOL_VERSION = "2" +TRANSCRIPT_SCHEMA_VERSION = "2" +RESULT_SCHEMA_VERSION = "2" + +REGISTER_PATH = "/v1/evaluator/workers/register" +CLAIM_PATH = "/v1/evaluator/assignments/claim" +TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" +HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" +RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" +WORKER_ID_HEADER = "X-FailproofAI-Worker-Id" +LEASE_GENERATION_HEADER = "X-FailproofAI-Lease-Generation" + +HEARTBEAT_INTERVAL_SECONDS = 30 +LEASE_DURATION_SECONDS = 120 +MAX_CLAIM_WAIT_SECONDS = 25 +MAX_ATTEMPTS = 5 + +MAX_CATALOG_DEFINITIONS = 100 +MAX_CLAIM_CAPACITY = 32 +MAX_TRANSCRIPT_BYTES = 25 * 1024 * 1024 +MAX_RESULTS_PER_RUN = 25 +MAX_EVAL_KEY_BYTES = 128 +MAX_DISPLAY_NAME_BYTES = 128 +MAX_VERSION_BYTES = 128 +MAX_WORKER_ID_BYTES = 128 +MAX_LABEL_BYTES = 64 +MAX_LABELS_PER_RESULT = 20 +MAX_SUMMARY_BYTES = 4 * 1024 +MAX_REASONING_BYTES = 16 * 1024 +MAX_UNIT_BYTES = 64 +MAX_DISPLAY_VALUE_BYTES = 256 +MAX_DESCRIPTION_BYTES = 1_000 +MAX_ERROR_CODE_BYTES = 64 +MAX_ERROR_MESSAGE_BYTES = 4 * 1024 + +ERROR_SPECS = { + "invalid_credentials": {"http_status": 401, "retryable": False}, + "instance_disabled": {"http_status": 403, "retryable": False}, + "assignment_not_found": {"http_status": 404, "retryable": False}, + "run_not_found": {"http_status": 404, "retryable": False}, + "catalog_mismatch": {"http_status": 409, "retryable": False}, + "lease_lost": {"http_status": 409, "retryable": False}, + "plan_conflict": {"http_status": 409, "retryable": False}, + "submission_conflict": {"http_status": 409, "retryable": False}, + "retry_budget_exhausted": {"http_status": 409, "retryable": False}, + "transcript_too_large": {"http_status": 413, "retryable": False}, + "invalid_request": {"http_status": 422, "retryable": False}, + "invalid_catalog": {"http_status": 422, "retryable": False}, + "unsupported_protocol_version": {"http_status": 426, "retryable": False}, + "internal_error": {"http_status": 500, "retryable": True}, +} + + +class ProtocolError(ValueError): + """A local or remote evaluator protocol contract violation.""" + + +class UnsupportedProtocolVersion(ProtocolError): + def __init__(self, received: str) -> None: + super().__init__( + f"unsupported evaluator protocol version {received!r}; " + f"supported major version is {PROTOCOL_VERSION}" + ) + self.received = received + + +def validate_protocol_version(version: str) -> None: + if version != PROTOCOL_VERSION: + raise UnsupportedProtocolVersion(version) + + +class EvaluatorKind(str, Enum): + MANAGED = "managed" + CUSTOMER = "customer" + + +class ResultKind(str, Enum): + SCORE = "score" + METRIC = "metric" + ASSERTION = "assertion" + + +class TerminalRunStatus(str, Enum): + SUCCEEDED = "succeeded" + FAILED = "failed" + TIMED_OUT = "timed_out" + CANCELLED = "cancelled" + + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +def _wire(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if hasattr(value, "__dataclass_fields__"): + return {key: _wire(item) for key, item in asdict(value).items()} + if isinstance(value, (list, tuple)): + return [_wire(item) for item in value] + if isinstance(value, dict): + return {key: _wire(item) for key, item in value.items()} + return value + + +class WireModel: + def to_wire(self) -> dict[str, Any]: + return _wire(self) + + +def _string(data: Mapping[str, Any], key: str) -> str: + value = data.get(key) + if not isinstance(value, str): + raise ProtocolError(f"{key} must be a string") + return value + + +def _integer(data: Mapping[str, Any], key: str) -> int: + value = data.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{key} must be an integer") + return value + + +def _list(data: Mapping[str, Any], key: str) -> list[Any]: + value = data.get(key) + if not isinstance(value, list): + raise ProtocolError(f"{key} must be an array") + return value + + +def _object(value: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ProtocolError(f"{field_name} must be an object") + return value + + +def _object_list(data: Mapping[str, Any], key: str) -> tuple[Mapping[str, Any], ...]: + return tuple( + _object(value, f"{key}[{index}]") + for index, value in enumerate(_list(data, key)) + ) + + +def _string_list(data: Mapping[str, Any], key: str) -> tuple[str, ...]: + values = _list(data, key) + for index, value in enumerate(values): + if not isinstance(value, str): + raise ProtocolError(f"{key}[{index}] must be a string") + return tuple(values) + + +def _enum(enum_type: type[_EnumT], data: Mapping[str, Any], key: str) -> _EnumT: + value = _string(data, key) + try: + return enum_type(value) + except ValueError as error: + allowed = ", ".join(repr(item.value) for item in enum_type) + raise ProtocolError(f"{key} must be one of {allowed}") from error + + +def _positive_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value <= 0: + raise ProtocolError(f"{key} must be greater than zero") + return value + + +def _nonnegative_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value < 0: + raise ProtocolError(f"{key} must not be negative") + return value + + +def _optional_string(data: Mapping[str, Any], key: str) -> str | None: + value = data.get(key) + if value is not None and not isinstance(value, str): + raise ProtocolError(f"{key} must be a string or null") + return value + + +@dataclass(frozen=True) +class CatalogDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> CatalogDefinition: + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class RegisterRequest(WireModel): + worker_id: str + sdk_version: str + catalog_revision: str + max_concurrency: int + definitions: tuple[CatalogDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + sdk_version=_string(data, "sdk_version"), + catalog_revision=_string(data, "catalog_revision"), + max_concurrency=_integer(data, "max_concurrency"), + definitions=tuple( + CatalogDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class RegisterResponse(WireModel): + evaluator_instance_id: str + evaluator_kind: EvaluatorKind + heartbeat_interval_seconds: int + lease_duration_seconds: int + claim_limit: int + disabled_definitions: tuple[str, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + evaluator_instance_id=_string(data, "evaluator_instance_id"), + evaluator_kind=_enum(EvaluatorKind, data, "evaluator_kind"), + heartbeat_interval_seconds=_integer(data, "heartbeat_interval_seconds"), + lease_duration_seconds=_integer(data, "lease_duration_seconds"), + claim_limit=_integer(data, "claim_limit"), + disabled_definitions=_string_list(data, "disabled_definitions"), + ) + + +@dataclass(frozen=True) +class ClaimRequest(WireModel): + worker_id: str + catalog_revision: str + capacity: int + wait_seconds: int + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + catalog_revision=_string(data, "catalog_revision"), + capacity=_integer(data, "capacity"), + wait_seconds=_integer(data, "wait_seconds"), + ) + + +@dataclass(frozen=True) +class Assignment(WireModel): + assignment_id: str + lease_generation: int + lease_expires_at: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + trigger_reason: str + event_count: int + transcript_url: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> Assignment: + return cls( + assignment_id=_string(data, "assignment_id"), + lease_generation=_positive_integer(data, "lease_generation"), + lease_expires_at=_string(data, "lease_expires_at"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + trigger_reason=_string(data, "trigger_reason"), + event_count=_nonnegative_integer(data, "event_count"), + transcript_url=_string(data, "transcript_url"), + ) + + +@dataclass(frozen=True) +class ClaimResponse(WireModel): + assignments: tuple[Assignment, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + tuple( + Assignment.from_wire(item) for item in _object_list(data, "assignments") + ) + ) + + +@dataclass(frozen=True) +class TranscriptEvent(WireModel): + id: str + ts: str + event_type: str + payload: Mapping[str, Any] + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> TranscriptEvent: + payload = data.get("payload") + if not isinstance(payload, Mapping): + raise ProtocolError("payload must be an object") + return cls( + id=_string(data, "id"), + ts=_string(data, "ts"), + event_type=_string(data, "event_type"), + payload=dict(payload), + ) + + +@dataclass(frozen=True) +class SessionTranscript(WireModel): + assignment_id: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + started_at: str + ended_at: str + event_count: int + events: tuple[TranscriptEvent, ...] + schema_version: str = TRANSCRIPT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SessionTranscript: + version = _string(data, "schema_version") + if version != TRANSCRIPT_SCHEMA_VERSION: + raise ProtocolError(f"unsupported transcript schema version {version!r}") + events = tuple( + TranscriptEvent.from_wire(item) for item in _object_list(data, "events") + ) + event_count = _nonnegative_integer(data, "event_count") + if event_count != len(events): + raise ProtocolError( + f"event_count is {event_count}, but transcript contains {len(events)} events" + ) + return cls( + assignment_id=_string(data, "assignment_id"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + started_at=_string(data, "started_at"), + ended_at=_string(data, "ended_at"), + event_count=event_count, + events=events, + ) + + def events_of_type(self, event_type: str) -> tuple[TranscriptEvent, ...]: + return tuple(event for event in self.events if event.event_type == event_type) + + def count(self, event_type: str) -> int: + return sum(event.event_type == event_type for event in self.events) + + +@dataclass(frozen=True) +class EvalSelection(WireModel): + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> EvalSelection: + return cls(_string(data, "eval_key"), _string(data, "eval_version")) + + +@dataclass(frozen=True) +class SkippedEval(WireModel): + eval_key: str + eval_version: str + reason_code: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SkippedEval: + return cls( + _string(data, "eval_key"), + _string(data, "eval_version"), + _string(data, "reason_code"), + ) + + +@dataclass(frozen=True) +class PlanRequest(WireModel): + worker_id: str + lease_generation: int + selected: tuple[EvalSelection, ...] = () + skipped: tuple[SkippedEval, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + selected=tuple( + EvalSelection.from_wire(item) for item in _object_list(data, "selected") + ), + skipped=tuple( + SkippedEval.from_wire(item) for item in _object_list(data, "skipped") + ), + ) + + +@dataclass(frozen=True) +class PlannedRun(WireModel): + evaluation_run_id: str + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + return cls( + _string(data, "evaluation_run_id"), + _string(data, "eval_key"), + _string(data, "eval_version"), + ) + + +@dataclass(frozen=True) +class PlanResponse(WireModel): + assignment_id: str + assignment_status: str + runs: tuple[PlannedRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + assignment_status=_string(data, "assignment_status"), + runs=tuple( + PlannedRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatRun(WireModel): + evaluation_run_id: str + state: str + progress: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRun: + progress = data.get("progress") + if progress is not None: + if isinstance(progress, bool) or not isinstance(progress, (int, float)): + raise ProtocolError("progress must be a number or null") + progress = float(progress) + if not math.isfinite(progress) or not 0 <= progress <= 1: + raise ProtocolError("progress must be finite and between 0 and 1") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + state=_string(data, "state"), + progress=progress, + ) + + +@dataclass(frozen=True) +class HeartbeatRequest(WireModel): + worker_id: str + lease_generation: int + runs: tuple[HeartbeatRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + runs=tuple( + HeartbeatRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatResponse(WireModel): + lease_expires_at: str + accepted_run_ids: tuple[str, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + lease_expires_at=_string(data, "lease_expires_at"), + accepted_run_ids=_string_list(data, "accepted_run_ids"), + ) + + +@dataclass(frozen=True) +class ResultItem(WireModel): + result_key: str + result_kind: ResultKind + numeric_value: float | None = None + bool_value: bool | None = None + text_value: str | None = None + unit: str = "" + display_value: str | None = None + description: str | None = None + reasoning: str | None = None + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultItem: + numeric = data.get("numeric_value") + if numeric is not None: + if isinstance(numeric, bool) or not isinstance(numeric, (int, float)): + raise ProtocolError("numeric_value must be a number or null") + numeric = float(numeric) + if not math.isfinite(numeric): + raise ProtocolError("numeric_value must be finite") + boolean = data.get("bool_value") + if boolean is not None and not isinstance(boolean, bool): + raise ProtocolError("bool_value must be a boolean or null") + return cls( + result_key=_string(data, "result_key"), + result_kind=_enum(ResultKind, data, "result_kind"), + numeric_value=numeric, + bool_value=boolean, + text_value=_optional_string(data, "text_value"), + unit=_string(data, "unit"), + display_value=_optional_string(data, "display_value"), + description=_optional_string(data, "description"), + reasoning=_optional_string(data, "reasoning"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class ResultRequest(WireModel): + submission_id: str + worker_id: str + lease_generation: int + status: TerminalRunStatus + started_at: str + finished_at: str + duration_ms: int + summary: str | None + results: tuple[ResultItem, ...] + error_code: str | None + error_message: str | None + protocol_version: str = PROTOCOL_VERSION + result_schema_version: str = RESULT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultRequest: + validate_protocol_version(_string(data, "protocol_version")) + result_schema_version = _string(data, "result_schema_version") + if result_schema_version != RESULT_SCHEMA_VERSION: + raise ProtocolError( + f"unsupported result schema version {result_schema_version!r}" + ) + return cls( + submission_id=_string(data, "submission_id"), + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + status=_enum(TerminalRunStatus, data, "status"), + started_at=_string(data, "started_at"), + finished_at=_string(data, "finished_at"), + duration_ms=_nonnegative_integer(data, "duration_ms"), + summary=_optional_string(data, "summary"), + results=tuple( + ResultItem.from_wire(item) for item in _object_list(data, "results") + ), + error_code=_optional_string(data, "error_code"), + error_message=_optional_string(data, "error_message"), + ) + + +@dataclass(frozen=True) +class ResultResponse(WireModel): + evaluation_run_id: str + submission_id: str + status: str + idempotent_replay: bool + result_count: int + result_checksum: str + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay") + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + submission_id=_string(data, "submission_id"), + status=_string(data, "status"), + idempotent_replay=replay, + result_count=_nonnegative_integer(data, "result_count"), + result_checksum=_string(data, "result_checksum"), + ) + + +@dataclass(frozen=True) +class RemoteError(WireModel): + code: str + message: str + retryable: bool + request_id: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RemoteError: + retryable = data.get("retryable") + if not isinstance(retryable, bool): + raise ProtocolError("retryable must be a boolean") + return cls( + code=_string(data, "code"), + message=_string(data, "message"), + retryable=retryable, + request_id=_string(data, "request_id"), + ) + + +@dataclass(frozen=True) +class ErrorResponse(WireModel): + error: RemoteError + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ErrorResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls(RemoteError.from_wire(_object(data.get("error"), "error"))) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py new file mode 100644 index 00000000..8a21402c --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -0,0 +1,563 @@ +"""Async worker state machine for Evaluator v2.""" + +from __future__ import annotations + +import asyncio +import inspect +import logging +import os +import socket +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from failproofai_sdk import __version__ +from failproofai_sdk.evaluator.authoring import ( + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + MAX_CLAIM_CAPACITY, + MAX_CLAIM_WAIT_SECONDS, + MAX_WORKER_ID_BYTES, + Assignment, + ClaimRequest, + EvalSelection, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + RegisterRequest, + ResultRequest, + SkippedEval, + TerminalRunStatus, +) + +logger = logging.getLogger("failproofai_sdk.evaluator") + + +def _utc_now() -> str: + return ( + datetime.now(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def _positive_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + if value <= 0: + raise ValueError(f"{name} must be greater than zero") + return value + + +@dataclass(frozen=True) +class WorkerConfig: + server_url: str + credential: str + worker_id: str + max_concurrency: int = 1 + claim_wait_seconds: int = 20 + request_timeout_seconds: int = 30 + drain_timeout_seconds: int = 60 + + @classmethod + def from_env(cls) -> WorkerConfig: + server_url = os.environ.get("FAILPROOFAI_EVALUATOR_URL", "").strip() + credential = os.environ.get("FAILPROOFAI_EVALUATOR_TOKEN", "").strip() + if not server_url: + raise ValueError("FAILPROOFAI_EVALUATOR_URL is required") + if not credential: + raise ValueError("FAILPROOFAI_EVALUATOR_TOKEN is required") + worker_id = os.environ.get("FAILPROOFAI_EVALUATOR_WORKER_ID", "").strip() + if not worker_id: + worker_id = f"{socket.gethostname()}-{os.getpid()}" + if len(worker_id.encode("utf-8")) > MAX_WORKER_ID_BYTES: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_WORKER_ID exceeds {MAX_WORKER_ID_BYTES} bytes" + ) + if any(ord(character) < 32 or ord(character) == 127 for character in worker_id): + raise ValueError( + "FAILPROOFAI_EVALUATOR_WORKER_ID must not contain control characters" + ) + config = cls( + server_url=server_url, + credential=credential, + worker_id=worker_id, + max_concurrency=_positive_int("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), + claim_wait_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", 20 + ), + request_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30 + ), + drain_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 + ), + ) + if config.max_concurrency > MAX_CLAIM_CAPACITY: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds {MAX_CLAIM_CAPACITY}" + ) + if config.claim_wait_seconds > MAX_CLAIM_WAIT_SECONDS: + raise ValueError( + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS exceeds " + f"{MAX_CLAIM_WAIT_SECONDS}" + ) + if config.request_timeout_seconds <= config.claim_wait_seconds: + raise ValueError( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS must exceed " + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS" + ) + return config + + +class WorkerRuntime: + def __init__( + self, + evaluator: Evaluator, + config: WorkerConfig, + *, + client: EvaluatorClient | None = None, + ) -> None: + self.evaluator = evaluator + self.config = config + self.client = client or EvaluatorClient( + base_url=config.server_url, + credential=config.credential, + timeout_seconds=config.request_timeout_seconds, + ) + self._stopping = asyncio.Event() + self._active: set[asyncio.Task[None]] = set() + self._heartbeat_interval = 30 + self._claim_limit = config.max_concurrency + self._lease_duration = 120 + self._disabled_definitions: set[str] = set() + self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + self._registered = False + self._last_server_contact: float | None = None + self._metric_lock = threading.Lock() + self._metrics: dict[str, int] = {} + + async def register(self) -> None: + try: + response = await self._call_client( + self.client.register, + RegisterRequest( + worker_id=self.config.worker_id, + sdk_version=__version__, + catalog_revision=self.evaluator.catalog_revision, + max_concurrency=self.config.max_concurrency, + definitions=self.evaluator.catalog(), + ), + ) + except Exception: + self._increment("registration_failure") + raise + self._heartbeat_interval = response.heartbeat_interval_seconds + self._lease_duration = response.lease_duration_seconds + self._claim_limit = min(self.config.max_concurrency, response.claim_limit) + if ( + self._heartbeat_interval <= 0 + or self._lease_duration <= self._heartbeat_interval + or self._claim_limit <= 0 + ): + self._increment("registration_failure") + raise RuntimeError( + "server returned invalid evaluator timing or claim limits" + ) + self._disabled_definitions = set(response.disabled_definitions) + self._registered = True + self._increment("registration_success") + + async def run_forever(self) -> None: + await self.register() + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + # With a transport error the server may have committed the + # lease while its response was lost. Waiting out that lease is + # what prevents a blind second claim from exceeding capacity. + await self._wait_or_stop( + float(self._lease_duration) if error.status is None else 1.0 + ) + continue + assignments = self._validated_assignments(response.assignments, capacity) + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + + await self.drain() + + async def run_once(self) -> int: + """Claim once and finish the returned assignments; useful for jobs/tests.""" + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=self._claim_limit, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + assignments = self._validated_assignments( + response.assignments, self._claim_limit + ) + self._increment("assignments_claimed", len(assignments)) + await asyncio.gather(*(self.process_assignment(item) for item in assignments)) + return len(assignments) + + def stop(self) -> None: + self._stopping.set() + + async def drain(self) -> None: + self._reap_finished() + if not self._active: + return + done, pending = await asyncio.wait( + self._active, timeout=self.config.drain_timeout_seconds + ) + for task in done: + self._consume_task(task) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + self._active.clear() + + async def process_assignment(self, assignment: Assignment) -> None: + session = await self._call_client( + self.client.transcript, + assignment, + worker_id=self.config.worker_id, + ) + if session.session_revision_id != assignment.session_revision_id: + raise RuntimeError("transcript session revision does not match assignment") + + selected: list[EvalDefinition] = [] + skipped: list[SkippedEval] = [] + for definition in self.evaluator.definitions: + if definition.eval_key in self._disabled_definitions: + skipped.append(self._skipped(definition, "disabled_by_server")) + self._increment("conditions_skipped") + continue + if definition.condition is None: + selected.append(definition) + continue + try: + condition = await self._invoke(definition.condition, session) + if isinstance(condition, ConditionResult): + applicable = condition.applicable + reason_code = condition.reason_code + elif isinstance(condition, bool): + applicable = condition + reason_code = "condition_false" + else: + raise TypeError("condition must return bool or ConditionResult") + except Exception as error: # noqa: BLE001 - isolates customer condition code + logger.warning( + "evaluator condition failed", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + skipped.append(self._skipped(definition, "condition_error")) + self._increment("conditions_skipped") + continue + if applicable: + selected.append(definition) + self._increment("conditions_selected") + else: + skipped.append(self._skipped(definition, reason_code)) + self._increment("conditions_skipped") + + plan = await self._call_client( + self.client.plan, + assignment.assignment_id, + PlanRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + selected=tuple( + EvalSelection(item.eval_key, item.eval_version) for item in selected + ), + skipped=tuple(skipped), + ), + ) + if plan.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned a plan for a different assignment") + expected_status = "planned" if selected else "skipped" + if plan.assignment_status != expected_status: + raise RuntimeError("server returned an inconsistent assignment status") + + definitions = {(item.eval_key, item.eval_version): item for item in selected} + run_definitions: list[tuple[str, EvalDefinition]] = [] + run_ids: set[str] = set() + for run in plan.runs: + if run.evaluation_run_id in run_ids: + raise RuntimeError("server returned a duplicate evaluation run id") + run_ids.add(run.evaluation_run_id) + definition = definitions.pop((run.eval_key, run.eval_version), None) + if definition is None: + raise RuntimeError("server returned an unrequested evaluation run") + run_definitions.append((run.evaluation_run_id, definition)) + if definitions: + raise RuntimeError("server omitted a selected evaluation run") + + tasks = { + run_id: asyncio.create_task( + self._execute_run(assignment, run_id, definition, session) + ) + for run_id, definition in run_definitions + } + heartbeat = asyncio.create_task(self._heartbeat(assignment, tasks)) + try: + outcomes = await asyncio.gather(*tasks.values(), return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise outcome + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _execute_run( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + async with self._eval_semaphore: + await self._execute_run_in_slot(assignment, run_id, definition, session) + + async def _execute_run_in_slot( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + started_at = _utc_now() + started = time.monotonic() + try: + invocation = self._invoke(definition.function, session) + result = ( + await asyncio.wait_for(invocation, timeout=definition.timeout_seconds) + if definition.timeout_seconds is not None + else await invocation + ) + if not isinstance(result, EvalResult): + raise TypeError("evaluation must return EvalResult") + items = result.result_items(definition.eval_key) + if not any( + item.result_key == definition.eval_key + and item.result_kind == definition.result_kind + for item in items + ): + raise ValueError( + "evaluation result does not contain its declared primary result" + ) + status = TerminalRunStatus.SUCCEEDED + summary = result.summary + error_code = None + error_message = None + except asyncio.TimeoutError: + await self._cancel_hook(definition, session) + items = () + status = TerminalRunStatus.TIMED_OUT + summary = None + error_code = "eval_timeout" + error_message = "evaluation exceeded its configured timeout" + except asyncio.CancelledError: + await self._cancel_hook(definition, session) + raise + except Exception as error: # noqa: BLE001 - converts customer eval failures + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + error_message = f"evaluation raised {type(error).__name__}" + + request = ResultRequest( + submission_id=str(uuid4()), + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + status=status, + started_at=started_at, + finished_at=_utc_now(), + duration_ms=max(0, round((time.monotonic() - started) * 1_000)), + summary=summary, + results=items, + error_code=error_code, + error_message=error_message, + ) + await self._call_client(self.client.submit_result, run_id, request) + self._increment(f"runs_{status.value}") + + async def _cancel_hook(self, definition: EvalDefinition, session: Any) -> None: + if definition.on_cancel is None: + return + try: + await self._invoke(definition.on_cancel, session) + except Exception as error: # noqa: BLE001 - cancellation hooks are customer code + logger.warning( + "evaluator cancellation hook failed", + extra={"error_type": type(error).__name__}, + ) + + async def _heartbeat( + self, + assignment: Assignment, + tasks: dict[str, asyncio.Task[None]], + ) -> None: + while True: + await asyncio.sleep(self._heartbeat_interval) + active = tuple( + HeartbeatRun(evaluation_run_id=run_id, state="running") + for run_id, task in tasks.items() + if not task.done() + ) + if not active: + return + try: + response = await self._call_client( + self.client.heartbeat, + HeartbeatRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + runs=active, + ), + ) + accepted = set(response.accepted_run_ids) + for run_id, task in tasks.items(): + if not task.done() and run_id not in accepted: + task.cancel() + except EvaluatorAPIError as error: + if error.code == "lease_lost": + self._increment("leases_lost") + for task in tasks.values(): + task.cancel() + return + logger.warning( + "evaluator heartbeat failed", + extra={ + "assignment_id": assignment.assignment_id, + "code": error.code, + }, + ) + self._increment("heartbeat_failures") + + @staticmethod + async def _invoke(function, session): + if inspect.iscoroutinefunction(function): + return await function(session) + result = await asyncio.to_thread(function, session) + if inspect.isawaitable(result): + return await result + return result + + @staticmethod + def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + def _reap_finished(self) -> None: + done = {task for task in self._active if task.done()} + self._active.difference_update(done) + for task in done: + self._consume_task(task) + + @staticmethod + def _validated_assignments( + assignments: tuple[Assignment, ...], capacity: int + ) -> tuple[Assignment, ...]: + if len(assignments) > capacity: + raise RuntimeError("server returned more assignments than requested") + assignment_ids = [item.assignment_id for item in assignments] + if len(assignment_ids) != len(set(assignment_ids)): + raise RuntimeError("server returned duplicate assignments") + return assignments + + @staticmethod + def _consume_task(task: asyncio.Task[None]) -> None: + try: + task.result() + except asyncio.CancelledError: + pass + except Exception: + logger.exception("evaluator assignment failed") + + async def _wait_for_progress(self) -> None: + if not self._active: + return + stop_task = asyncio.create_task(self._stopping.wait()) + try: + await asyncio.wait( + (*self._active, stop_task), return_when=asyncio.FIRST_COMPLETED + ) + finally: + if not stop_task.done(): + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + + async def _wait_or_stop(self, seconds: float) -> None: + try: + await asyncio.wait_for(self._stopping.wait(), timeout=seconds) + except asyncio.TimeoutError: + pass + + async def _call_client(self, function, *args, **kwargs): + result = await asyncio.to_thread(function, *args, **kwargs) + self._last_server_contact = time.monotonic() + return result + + def _increment(self, name: str, amount: int = 1) -> None: + with self._metric_lock: + self._metrics[name] = self._metrics.get(name, 0) + amount + + def metrics(self) -> dict[str, int]: + with self._metric_lock: + return dict(self._metrics) + + def is_ready(self) -> bool: + if ( + self._stopping.is_set() + or not self._registered + or self._last_server_contact is None + ): + return False + return time.monotonic() - self._last_server_contact <= max( + float(self._lease_duration), 60.0 + ) diff --git a/sdk/python/tests/fixtures/evaluator_v2/README.md b/sdk/python/tests/fixtures/evaluator_v2/README.md new file mode 100644 index 00000000..e57c93bd --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/README.md @@ -0,0 +1,28 @@ +# Evaluator v2 contract fixtures + +`contract.json` is the Checkpoint 0 wire contract shared by the Rust server and +the zero-dependency Python SDK. The matching copy lives at +`server/tests/fixtures/evaluator_v2/contract.json` in the `agenteye` repository. +Change both copies together. + +Contract rules: + +- The only accepted protocol major is the exact string `"2"`. Unsupported + majors return `426 unsupported_protocol_version`. +- Unknown JSON fields are ignored so either side may add optional fields within + major version 2. Removing, renaming, or changing the meaning of a field needs + a new major version. +- Worker payloads never carry authoritative tenant or evaluator-instance + identity. The server derives those from the credential and leased record. +- `lease_generation` is the fencing token. `409 lease_lost` is terminal for the + affected local execution; the SDK must stop heartbeating or submitting it. +- `submission_id` is an idempotency key. Replaying identical content succeeds; + reusing it for different content returns `409 submission_conflict`. +- Transcript overflow is terminal in v2 (`413 transcript_too_large`); the server + never silently truncates the evaluated input. +- Only errors marked `retryable` may be retried automatically. HTTP method alone + is not enough to decide whether a protocol operation is safe to replay. + +The timing and payload limits in the fixture are normative defaults. A register +response may lower the worker's effective concurrency, heartbeat interval, or +lease duration, but may not raise a client-side payload bound. diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json new file mode 100644 index 00000000..aa0cf275 --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -0,0 +1,225 @@ +{ + "fixture_revision": "evaluator-v2-2026-08-28.2", + "protocol": { + "supported_major_versions": ["2"], + "transcript_schema_version": "2", + "result_schema_version": "2" + }, + "http": { + "register": "/v1/evaluator/workers/register", + "claim": "/v1/evaluator/assignments/claim", + "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "plan": "/v1/evaluator/assignments/{assignment_id}/plan", + "heartbeat": "/v1/evaluator/runs/heartbeat", + "result": "/v1/evaluator/runs/{evaluation_run_id}/result", + "worker_id_header": "X-FailproofAI-Worker-Id", + "lease_generation_header": "X-FailproofAI-Lease-Generation" + }, + "timing": { + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "max_claim_wait_seconds": 25, + "max_attempts": 5 + }, + "limits": { + "max_catalog_definitions": 100, + "max_claim_capacity": 32, + "max_transcript_bytes": 26214400, + "max_results_per_run": 25, + "max_eval_key_bytes": 128, + "max_display_name_bytes": 128, + "max_version_bytes": 128, + "max_worker_id_bytes": 128, + "max_label_bytes": 64, + "max_labels_per_result": 20, + "max_summary_bytes": 4096, + "max_reasoning_bytes": 16384, + "max_unit_bytes": 64, + "max_display_value_bytes": 256, + "max_description_bytes": 1000, + "max_error_code_bytes": 64, + "max_error_message_bytes": 4096 + }, + "errors": { + "invalid_credentials": {"http_status": 401, "retryable": false}, + "instance_disabled": {"http_status": 403, "retryable": false}, + "assignment_not_found": {"http_status": 404, "retryable": false}, + "run_not_found": {"http_status": 404, "retryable": false}, + "catalog_mismatch": {"http_status": 409, "retryable": false}, + "lease_lost": {"http_status": 409, "retryable": false}, + "plan_conflict": {"http_status": 409, "retryable": false}, + "submission_conflict": {"http_status": 409, "retryable": false}, + "retry_budget_exhausted": {"http_status": 409, "retryable": false}, + "transcript_too_large": {"http_status": 413, "retryable": false}, + "invalid_request": {"http_status": 422, "retryable": false}, + "invalid_catalog": {"http_status": 422, "retryable": false}, + "unsupported_protocol_version": {"http_status": 426, "retryable": false}, + "internal_error": {"http_status": 500, "retryable": true} + }, + "samples": { + "register_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "sdk_version": "0.0.1b2", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "max_concurrency": 4, + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"] + } + ] + }, + "register_response": { + "protocol_version": "2", + "evaluator_instance_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23101", + "evaluator_kind": "customer", + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "claim_limit": 4, + "disabled_definitions": [] + }, + "claim_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "capacity": 2, + "wait_seconds": 20 + }, + "claim_response": { + "protocol_version": "2", + "assignments": [ + { + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "lease_generation": 3, + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "trigger_reason": "agent_end", + "event_count": 42, + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript" + } + ] + }, + "transcript_response": { + "schema_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "started_at": "2026-08-28T11:58:00.000000Z", + "ended_at": "2026-08-28T12:00:00.000000Z", + "event_count": 2, + "events": [ + { + "id": "evt-tool-1", + "ts": "2026-08-28T11:59:00.000000Z", + "event_type": "tool_use", + "payload": {"tool_name": "search"} + }, + { + "id": "evt-end-1", + "ts": "2026-08-28T12:00:00.000000Z", + "event_type": "agent_end", + "payload": {"summary": "Done"} + } + ] + }, + "plan_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "selected": [ + {"eval_key": "tool_efficiency", "eval_version": "1.2.0"} + ], + "skipped": [ + { + "eval_key": "answer_groundedness", + "eval_version": "2.1.0", + "reason_code": "no_retrieval_events" + } + ] + }, + "plan_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "assignment_status": "planned", + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "eval_key": "tool_efficiency", + "eval_version": "1.2.0" + } + ] + }, + "heartbeat_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "state": "running", + "progress": 0.5 + } + ] + }, + "heartbeat_response": { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": ["018f47a8-7c1d-7e21-a22a-79f7a4d23103"] + }, + "result_request": { + "protocol_version": "2", + "result_schema_version": "2", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "status": "succeeded", + "started_at": "2026-08-28T12:00:10.000000Z", + "finished_at": "2026-08-28T12:00:10.812000Z", + "duration_ms": 812, + "summary": "Used a compact tool set without retries.", + "results": [ + { + "result_key": "tool_efficiency", + "result_kind": "score", + "numeric_value": 0.92, + "bool_value": true, + "text_value": null, + "unit": "ratio", + "display_value": "92%", + "description": "Distinct tools divided by total tool calls", + "reasoning": "3 distinct tools across 3 calls", + "labels": ["tools", "deterministic"] + } + ], + "error_code": null, + "error_message": null + }, + "result_response": { + "protocol_version": "2", + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "status": "committed", + "idempotent_replay": false, + "result_count": 1, + "result_checksum": "sha256:8cbd34f2d95d" + }, + "error_response": { + "protocol_version": "2", + "error": { + "code": "lease_lost", + "message": "The assignment lease is no longer owned by this worker.", + "retryable": false, + "request_id": "req-018f47a8" + } + } + } +} diff --git a/sdk/python/tests/test_evaluator_authoring.py b/sdk/python/tests/test_evaluator_authoring.py new file mode 100644 index 00000000..578b1220 --- /dev/null +++ b/sdk/python/tests/test_evaluator_authoring.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +import math + +import pytest + +from failproofai_sdk.evaluator import ( + Assertion, + EvalResult, + Evaluator, + Metric, + ResultKind, + Score, +) + + +def test_catalog_is_stable_across_registration_order(): + first = Evaluator(name="acme", version="2026.08.1") + second = Evaluator(name="acme", version="2026.08.1") + + @first.eval("zeta_check", version="1", labels=["z", "a"]) + def first_zeta(session): + return EvalResult(score=Score(1)) + + @first.eval("alpha_check", version="1") + def first_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("alpha_check", version="1") + def second_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("zeta_check", version="1", labels=["a", "z"]) + def second_zeta(session): + return EvalResult(score=Score(1)) + + assert first.catalog_revision == second.catalog_revision + assert [item.eval_key for item in first.catalog()] == ["alpha_check", "zeta_check"] + + +def test_duplicate_eval_keys_are_rejected_even_when_versions_differ(): + evaluator = Evaluator(name="acme", version="1") + + @evaluator.eval("quality", version="1") + def quality_v1(session): + return EvalResult(score=Score(1)) + + with pytest.raises(ValueError, match="duplicate eval key"): + + @evaluator.eval("quality", version="2") + def quality_v2(session): + return EvalResult(score=Score(1)) + + +@pytest.mark.parametrize("value", [-0.01, 1.01, math.nan, math.inf]) +def test_scores_are_finite_ratios(value): + with pytest.raises(ValueError): + Score(value) + + +def test_result_presentation_fields_are_bounded_before_networking(): + with pytest.raises(ValueError, match="unit is 65 bytes"): + Metric(1, unit="u" * 65) + with pytest.raises(ValueError, match="display value is 257 bytes"): + Score(1, display_value="x" * 257) + with pytest.raises(ValueError, match="description is 1001 bytes"): + Assertion(True, description="x" * 1001) + + +def test_eval_result_expands_to_typed_long_form_rows(): + result = EvalResult( + score=Score(0.75, passed=True, unit="ratio"), + metrics={"call_count": Metric(4, unit="calls")}, + assertions={"had_output": Assertion(True)}, + reasoning="Three useful calls out of four.", + labels=("tools",), + ) + + items = result.result_items("tool_efficiency") + assert [item.result_kind for item in items] == [ + ResultKind.SCORE, + ResultKind.METRIC, + ResultKind.ASSERTION, + ] + assert items[0].reasoning == "Three useful calls out of four." + assert items[1].numeric_value == 4 + assert items[2].bool_value is True + + +def test_empty_eval_result_is_rejected_when_serialized(): + with pytest.raises(ValueError, match="must contain"): + EvalResult().result_items("quality") + + +def test_result_keys_must_be_unique_across_kinds(): + result = EvalResult(score=Score(1), metrics={"quality": 1}) + with pytest.raises(ValueError, match="result keys must be unique"): + result.result_items("quality") + + +def test_sync_and_async_functions_share_one_call_path(): + async def async_eval(session): + return EvalResult(score=Score(1)) + + def sync_eval(session): + return EvalResult(score=Score(0.5)) + + async def exercise(): + sync_result = await Evaluator.call(sync_eval, None) + async_result = await Evaluator.call(async_eval, None) + return sync_result, async_result + + sync_result, async_result = asyncio.run(exercise()) + assert sync_result.score.value == 0.5 + assert async_result.score.value == 1 + + +def test_keys_are_machine_safe_and_versions_are_explicit(): + evaluator = Evaluator(name="acme", version="1") + with pytest.raises(ValueError, match="must match"): + evaluator.eval("Not Safe", version="1") + with pytest.raises(ValueError, match="must not be empty"): + evaluator.eval("safe", version="") diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py new file mode 100644 index 00000000..7513e306 --- /dev/null +++ b/sdk/python/tests/test_evaluator_client.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError, URLError + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvaluatorAPIError, + EvaluatorClient, + ResultRequest, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class Response: + def __init__(self, body): + self.body = json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def read(self, amount): + return self.body[:amount] + + +class RawResponse(Response): + def __init__(self, body: bytes): + self.body = body + + +def test_claim_sends_bearer_auth_and_does_not_retry(): + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + raise URLError("offline") + + client = EvaluatorClient( + base_url="https://cloud.example/api/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + with pytest.raises(EvaluatorAPIError, match="transport_error"): + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + + assert len(calls) == 1 + request, timeout = calls[0] + assert request.full_url == "https://cloud.example/v1/evaluator/assignments/claim" + assert request.get_header("Authorization") == "Bearer secret" + assert timeout == 30 + + +def test_idempotent_result_submission_retries_transport_failure(): + samples = _samples() + calls = 0 + + def opener(request, timeout): + nonlocal calls + calls += 1 + if calls == 1: + raise URLError("reset") + return Response(samples["result_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + response = client.submit_result( + samples["result_response"]["evaluation_run_id"], + ResultRequest.from_wire(samples["result_request"]), + ) + assert response.status == "committed" + assert calls == 2 + + +def test_transcript_url_cannot_exfiltrate_the_worker_credential(): + sample = _samples()["claim_response"]["assignments"][0] + assignment = Assignment.from_wire( + {**sample, "transcript_url": "https://evil.test/read"} + ) + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda *_args, **_kwargs: pytest.fail("network must not be reached"), + ) + with pytest.raises(EvaluatorAPIError, match="outside the configured API origin"): + client.transcript(assignment, worker_id="worker") + + +def test_machine_error_envelope_controls_retryability(): + body = json.dumps(_samples()["error_response"]).encode() + + def opener(request, timeout): + raise HTTPError(request.full_url, 409, "Conflict", {}, io.BytesIO(body)) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == "lease_lost" + assert caught.value.status == 409 + assert caught.value.retryable is False + assert caught.value.request_id == "req-018f47a8" + + +@pytest.mark.parametrize( + ("body", "code"), + [ + (b"not-json", "invalid_response"), + (b"[]", "invalid_response"), + ( + b"{" + b'"padding":"' + b"x" * (2 * 1024 * 1024) + b'"}', + "response_too_large", + ), + ], +) +def test_malformed_or_oversized_server_responses_fail_closed(body, code): + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda request, timeout: RawResponse(body), + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == code + assert caught.value.retryable is False + + +def test_transcript_identity_is_sent_as_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["transcript_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + client.transcript(assignment, worker_id="worker-7") + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_constructor_rejects_unsafe_or_incomplete_configuration(): + with pytest.raises(ValueError, match="absolute"): + EvaluatorClient(base_url="localhost:8080", credential="secret") + with pytest.raises(ValueError, match="credential"): + EvaluatorClient(base_url="https://cloud.example", credential="") + with pytest.raises(ValueError, match="control characters"): + EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") + + +def test_protocol_redirect_does_not_forward_the_bearer_credential(): + exfiltration_attempts = [] + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + exfiltration_attempts.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): + return + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + sink_thread = threading.Thread(target=sink.serve_forever, daemon=True) + sink_thread.start() + + location = f"http://127.0.0.1:{sink.server_address[1]}/steal" + + class Redirector(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(307) + self.send_header("Location", location) + self.end_headers() + + def log_message(self, format, *args): + return + + redirector = ThreadingHTTPServer(("127.0.0.1", 0), Redirector) + redirector_thread = threading.Thread(target=redirector.serve_forever, daemon=True) + redirector_thread.start() + try: + client = EvaluatorClient( + base_url=f"http://127.0.0.1:{redirector.server_address[1]}", + credential="must-not-leak", + max_retries=0, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 0)) + assert caught.value.status == 307 + assert exfiltration_attempts == [] + finally: + redirector.shutdown() + redirector.server_close() + redirector_thread.join(timeout=5) + sink.shutdown() + sink.server_close() + sink_thread.join(timeout=5) diff --git a/sdk/python/tests/test_evaluator_example.py b/sdk/python/tests/test_evaluator_example.py new file mode 100644 index 00000000..e85a545e --- /dev/null +++ b/sdk/python/tests/test_evaluator_example.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ConditionResult + + +def _example_module(): + path = Path(__file__).parents[1] / "examples" / "evaluator_worker.py" + spec = importlib.util.spec_from_file_location("evaluator_worker_example", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_example_registers_deterministic_and_async_evals(monkeypatch): + monkeypatch.delenv("EXAMPLE_JUDGE_URL", raising=False) + module = _example_module() + definitions = {item.eval_key: item for item in module.app.definitions} + assert set(definitions) == {"answer_relevance", "tool_efficiency"} + assert definitions["answer_relevance"].eval_version == "judge-api-v1" + + skipped = definitions["answer_relevance"].condition(None) + assert skipped == ConditionResult(False, "judge_not_configured") + + +def test_example_rejects_non_http_judge_urls(monkeypatch): + module = _example_module() + monkeypatch.setenv("EXAMPLE_JUDGE_URL", "file:///etc/passwd") + with pytest.raises(ValueError, match="absolute http"): + module._call_judge("question", "answer") diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py new file mode 100644 index 00000000..475e0bf3 --- /dev/null +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -0,0 +1,631 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit +from uuid import UUID, uuid5 + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvalResult, + EvalSelection, + Evaluator, + EvaluatorAPIError, + EvaluatorClient, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + ResultItem, + ResultKind, + ResultRequest, + Score, + TerminalRunStatus, + WorkerConfig, + WorkerRuntime, +) + +_NAMESPACE = UUID("4d592d9c-aed4-4f07-9b2d-e14963399df6") + + +class ProtocolState: + def __init__(self) -> None: + self.lock = threading.Lock() + self.base_url = "" + self.instances = { + "customer-a-token": ("instance-customer-a", "customer", "org-a"), + "customer-b-token": ("instance-customer-b", "customer", "org-b"), + "managed-token": ("instance-managed", "managed", None), + } + self.registrations: dict[str, dict] = {} + self.assignments: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + self.result_attempts = 0 + self.result_commits = 0 + self.last_result_body: dict | None = None + self.drop_first_result_response = False + + def add_assignment(self, name: str, *, token: str, org: str) -> str: + assignment_id = str(uuid5(_NAMESPACE, name)) + self.assignments[assignment_id] = { + "token": token, + "org": org, + "status": "available", + "worker_id": None, + "lease_generation": 0, + "expired": False, + "session_id": f"session-{name}", + "session_revision_id": f"revision-{name}", + } + return assignment_id + + def expire(self, assignment_id: str) -> None: + with self.lock: + self.assignments[assignment_id]["expired"] = True + + +class ProtocolServer: + def __init__(self, state: ProtocolState) -> None: + self.state = state + handler = _handler_for(state) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.server.daemon_threads = True + state.base_url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> ProtocolServer: # noqa: PYI034 - Python 3.10 lacks Self + self.thread.start() + return self + + def __exit__(self, *_args) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +def _handler_for(state: ProtocolState): + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + body = self._body() + path = urlsplit(self.path).path + if path == "/v1/evaluator/workers/register": + state.registrations[token] = body + instance_id, kind, _org = state.instances[token] + self._json( + 200, + { + "protocol_version": "2", + "evaluator_instance_id": instance_id, + "evaluator_kind": kind, + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "claim_limit": body["max_concurrency"], + "disabled_definitions": [], + }, + ) + return + if path == "/v1/evaluator/assignments/claim": + self._claim(token, body) + return + if path.endswith("/plan"): + self._plan(token, path.split("/")[-2], body) + return + if path == "/v1/evaluator/runs/heartbeat": + self._heartbeat(token, body) + return + if path.endswith("/result"): + self._result(token, path.split("/")[-2], body) + return + self._error(404, "assignment_not_found", False) + + def do_GET(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + path = urlsplit(self.path).path + if path.endswith("/transcript"): + self._transcript(token, path.split("/")[-2]) + return + self._error(404, "assignment_not_found", False) + + def _claim(self, token: str, body: dict) -> None: + claimed = [] + with state.lock: + for assignment_id, item in state.assignments.items(): + if len(claimed) >= body["capacity"]: + break + if item["token"] != token: + continue + if item["status"] in {"leased", "planned"} and not item["expired"]: + continue + if item["status"] not in {"available", "leased", "planned"}: + continue + item["status"] = "leased" + item["expired"] = False + item["worker_id"] = body["worker_id"] + item["lease_generation"] += 1 + claimed.append(self._assignment_wire(assignment_id, item)) + self._json(200, {"protocol_version": "2", "assignments": claimed}) + + def _transcript(self, token: str, assignment_id: str) -> None: + item = self._leased_assignment( + token, + assignment_id, + self.headers.get("X-FailproofAI-Worker-Id"), + self.headers.get("X-FailproofAI-Lease-Generation"), + ) + if item is None: + return + self._json( + 200, + { + "schema_version": "2", + "assignment_id": assignment_id, + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "started_at": "2026-08-28T12:00:00.000000Z", + "ended_at": "2026-08-28T12:00:01.000000Z", + "event_count": 1, + "events": [ + { + "id": "event-1", + "ts": "2026-08-28T12:00:00.500000Z", + "event_type": "model_response", + "payload": {"content": "done"}, + } + ], + }, + ) + + def _plan(self, token: str, assignment_id: str, body: dict) -> None: + item = self._leased_assignment( + token, assignment_id, body["worker_id"], body["lease_generation"] + ) + if item is None: + return + runs = [] + with state.lock: + for selected in body["selected"]: + run_id = str( + uuid5( + _NAMESPACE, + f"{assignment_id}:{selected['eval_key']}:{selected['eval_version']}", + ) + ) + run = state.runs.setdefault( + run_id, + { + "token": token, + "assignment_id": assignment_id, + "worker_id": body["worker_id"], + "lease_generation": body["lease_generation"], + "submission_id": None, + "checksum": None, + }, + ) + if run["submission_id"] is None: + run["worker_id"] = body["worker_id"] + run["lease_generation"] = body["lease_generation"] + runs.append({"evaluation_run_id": run_id, **selected}) + item["status"] = "planned" if runs else "skipped" + self._json( + 200, + { + "protocol_version": "2", + "assignment_id": assignment_id, + "assignment_status": item["status"], + "runs": runs, + }, + ) + + def _heartbeat(self, token: str, body: dict) -> None: + accepted = [] + with state.lock: + for requested in body["runs"]: + run = state.runs.get(requested["evaluation_run_id"]) + assignment = ( + state.assignments.get(run["assignment_id"]) + if run is not None + else None + ) + if ( + run is not None + and assignment is not None + and run["token"] == token + and run["worker_id"] == body["worker_id"] + and run["lease_generation"] == body["lease_generation"] + and assignment["worker_id"] == body["worker_id"] + and assignment["lease_generation"] == body["lease_generation"] + and not assignment["expired"] + ): + accepted.append(requested["evaluation_run_id"]) + if not accepted: + self._error(409, "lease_lost", False) + return + self._json( + 200, + { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": accepted, + }, + ) + + def _result(self, token: str, run_id: str, body: dict) -> None: + with state.lock: + run = state.runs.get(run_id) + if run is None or run["token"] != token: + self._error(404, "run_not_found", False) + return + if ( + run["worker_id"] != body["worker_id"] + or run["lease_generation"] != body["lease_generation"] + ): + self._error(409, "lease_lost", False) + return + checksum = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if run["submission_id"] is not None: + if ( + run["submission_id"] != body["submission_id"] + or run["checksum"] != checksum + ): + self._error(409, "submission_conflict", False) + return + replay = True + else: + run["submission_id"] = body["submission_id"] + run["checksum"] = checksum + state.result_commits += 1 + state.last_result_body = body + replay = False + state.result_attempts += 1 + drop = state.drop_first_result_response and state.result_attempts == 1 + if drop: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self._json( + 200, + { + "protocol_version": "2", + "evaluation_run_id": run_id, + "submission_id": body["submission_id"], + "status": "committed", + "idempotent_replay": replay, + "result_count": len(body["results"]), + "result_checksum": checksum, + }, + ) + + def _leased_assignment( + self, + token: str, + assignment_id: str, + worker_id: str | None, + generation: str | int | None, + ) -> dict | None: + with state.lock: + item = state.assignments.get(assignment_id) + if item is None or item["token"] != token: + self._error(404, "assignment_not_found", False) + return None + try: + generation = int(generation) if generation is not None else None + except ValueError: + generation = None + if ( + item["status"] != "leased" + or item["expired"] + or item["worker_id"] != worker_id + or item["lease_generation"] != generation + ): + self._error(409, "lease_lost", False) + return None + return item + + def _assignment_wire(self, assignment_id: str, item: dict) -> dict: + return { + "assignment_id": assignment_id, + "lease_generation": item["lease_generation"], + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "trigger_reason": "agent_end", + "event_count": 1, + "transcript_url": ( + f"{state.base_url}/v1/evaluator/assignments/" + f"{assignment_id}/transcript" + ), + } + + def _token(self) -> str | None: + value = self.headers.get("Authorization", "") + return ( + value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + ) + + def _body(self) -> dict: + size = int(self.headers.get("Content-Length", "0")) + return json.loads(self.rfile.read(size)) + + def _error(self, status: int, code: str, retryable: bool) -> None: + self._json( + status, + { + "protocol_version": "2", + "error": { + "code": code, + "message": code.replace("_", " "), + "retryable": retryable, + "request_id": "request-e2e", + }, + }, + ) + + def _json(self, status: int, value: dict) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: + return + + return Handler + + +def _client(state: ProtocolState, token: str) -> EvaluatorClient: + return EvaluatorClient( + base_url=state.base_url, + credential=token, + max_retries=2, + sleeper=lambda _seconds: None, + ) + + +def _claim(client: EvaluatorClient, worker_id: str): + return client.claim( + ClaimRequest( + worker_id=worker_id, + catalog_revision="sha256:" + "a" * 64, + capacity=1, + wait_seconds=0, + ) + ) + + +def test_real_http_worker_survives_lost_result_response_without_duplicate_commit(): + state = ProtocolState() + state.add_assignment("runtime", token="customer-a-token", org="org-a") + state.drop_first_result_response = True + evaluator = Evaluator(name="e2e", version="1") + + @evaluator.eval("completion_present", version="1") + def completion_present(session): + return EvalResult(score=Score(float(bool(session.events)))) + + with ProtocolServer(state): + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url=state.base_url, + credential="customer-a-token", + worker_id="worker-a", + claim_wait_seconds=1, + ), + client=_client(state, "customer-a-token"), + ) + + async def run(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(run()) == 1 + + run_id = next(iter(state.runs)) + committed = ResultRequest.from_wire(state.last_result_body) + replay = runtime.client.submit_result(run_id, committed) + assert replay.idempotent_replay is True + + with pytest.raises(EvaluatorAPIError) as caught: + runtime.client.submit_result( + run_id, + replace(committed, summary="different content"), + ) + assert caught.value.code == "submission_conflict" + + assert state.result_attempts == 3 + assert state.result_commits == 1 + assert len(state.runs) == 1 + assert next(iter(state.runs.values()))["submission_id"] is not None + assert runtime.metrics()["runs_succeeded"] == 1 + + +def test_two_workers_racing_receive_one_unique_lease(): + state = ProtocolState() + assignment_id = state.add_assignment("race", token="customer-a-token", org="org-a") + with ProtocolServer(state): + first = _client(state, "customer-a-token") + second = _client(state, "customer-a-token") + with ThreadPoolExecutor(max_workers=2) as executor: + responses = list( + executor.map( + lambda pair: _claim(*pair), + [(first, "worker-a"), (second, "worker-b")], + ) + ) + + claimed = [item for response in responses for item in response.assignments] + assert [item.assignment_id for item in claimed] == [assignment_id] + assert state.assignments[assignment_id]["lease_generation"] == 1 + + +def test_expired_lease_is_reclaimed_and_stale_worker_is_fenced(): + state = ProtocolState() + assignment_id = state.add_assignment( + "reclaim", token="customer-a-token", org="org-a" + ) + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + + assert second.lease_generation == first.lease_generation + 1 + with pytest.raises(EvaluatorAPIError) as caught: + client.transcript(first, worker_id="worker-a") + assert caught.value.code == "lease_lost" + assert caught.value.retryable is False + + with pytest.raises(EvaluatorAPIError) as caught: + client.heartbeat( + HeartbeatRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + runs=(HeartbeatRun(plan.runs[0].evaluation_run_id, "running"),), + ) + ) + assert caught.value.code == "lease_lost" + + +def test_replacement_worker_finishes_after_forced_worker_loss(): + state = ProtocolState() + assignment_id = state.add_assignment( + "forced-worker-loss", token="customer-a-token", org="org-a" + ) + result_sample = ResultRequest( + submission_id=str(uuid5(_NAMESPACE, "forced-worker-loss-result")), + worker_id="worker-a", + lease_generation=1, + status=TerminalRunStatus.SUCCEEDED, + started_at="2026-08-28T12:00:10.000000Z", + finished_at="2026-08-28T12:00:10.100000Z", + duration_ms=100, + summary="Replacement worker completed the evaluation.", + results=( + ResultItem( + result_key="quality", + result_kind=ResultKind.SCORE, + numeric_value=1.0, + ), + ), + error_code=None, + error_message=None, + ) + + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + first_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + + # Worker A disappears after planning. Its lease expires and worker B + # reclaims the same logical assignment and deterministic run. + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + second_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-b", + lease_generation=second.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + assert ( + second_plan.runs[0].evaluation_run_id + == first_plan.runs[0].evaluation_run_id + ) + + stale_result = replace( + result_sample, + worker_id="worker-a", + lease_generation=first.lease_generation, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.submit_result(first_plan.runs[0].evaluation_run_id, stale_result) + assert caught.value.code == "lease_lost" + + replacement_result = replace( + result_sample, + worker_id="worker-b", + lease_generation=second.lease_generation, + ) + committed = client.submit_result( + second_plan.runs[0].evaluation_run_id, replacement_result + ) + assert committed.status == "committed" + assert committed.idempotent_replay is False + + assert state.result_attempts == 1 + assert state.result_commits == 1 + + +def test_customer_tenants_are_isolated_and_managed_worker_coexists(): + state = ProtocolState() + customer_id = state.add_assignment( + "customer", token="customer-a-token", org="org-a" + ) + managed_id = state.add_assignment("managed", token="managed-token", org="org-b") + with ProtocolServer(state): + customer_a = _client(state, "customer-a-token") + customer_b = _client(state, "customer-b-token") + managed = _client(state, "managed-token") + + customer_assignment = _claim(customer_a, "worker-a").assignments[0] + assert customer_assignment.assignment_id == customer_id + assert _claim(customer_b, "worker-b").assignments == () + assert ( + _claim(managed, "worker-managed").assignments[0].assignment_id == managed_id + ) + + stolen = Assignment( + assignment_id=customer_assignment.assignment_id, + lease_generation=customer_assignment.lease_generation, + lease_expires_at=customer_assignment.lease_expires_at, + session_id=customer_assignment.session_id, + session_revision_id=customer_assignment.session_revision_id, + agent_id=customer_assignment.agent_id, + environment=customer_assignment.environment, + trigger_reason=customer_assignment.trigger_reason, + event_count=customer_assignment.event_count, + transcript_url=customer_assignment.transcript_url, + ) + with pytest.raises(EvaluatorAPIError) as caught: + customer_b.transcript(stolen, worker_id="worker-a") + assert caught.value.status == 404 + assert caught.value.code == "assignment_not_found" diff --git a/sdk/python/tests/test_evaluator_main.py b/sdk/python/tests/test_evaluator_main.py new file mode 100644 index 00000000..c846abdc --- /dev/null +++ b/sdk/python/tests/test_evaluator_main.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sys + +import pytest + +from failproofai_sdk.evaluator import Evaluator +from failproofai_sdk.evaluator.__main__ import load_evaluator + + +def test_module_loader_defaults_to_app(tmp_path, monkeypatch): + (tmp_path / "my_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "app = Evaluator(name='example', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("my_evals") + finally: + sys.modules.pop("my_evals", None) + assert isinstance(loaded, Evaluator) + assert loaded.name == "example" + + +def test_module_loader_supports_an_explicit_attribute(tmp_path, monkeypatch): + (tmp_path / "custom_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "worker = Evaluator(name='custom', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("custom_evals:worker") + finally: + sys.modules.pop("custom_evals", None) + assert loaded.name == "custom" + + +def test_module_loader_rejects_the_wrong_object_type(tmp_path, monkeypatch): + (tmp_path / "not_evals.py").write_text("app = object()\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + with pytest.raises(TypeError, match="not Evaluator"): + load_evaluator("not_evals") + finally: + sys.modules.pop("not_evals", None) diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py new file mode 100644 index 00000000..a7fd18ee --- /dev/null +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + UnsupportedProtocolVersion, + protocol, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _contract(): + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("sample", "model"), + [ + ("register_request", RegisterRequest), + ("register_response", RegisterResponse), + ("claim_request", ClaimRequest), + ("claim_response", ClaimResponse), + ("transcript_response", SessionTranscript), + ("plan_request", PlanRequest), + ("plan_response", PlanResponse), + ("heartbeat_request", HeartbeatRequest), + ("heartbeat_response", HeartbeatResponse), + ("result_request", ResultRequest), + ("result_response", ResultResponse), + ("error_response", ErrorResponse), + ], +) +def test_golden_messages_round_trip(sample, model): + wire = _contract()["samples"][sample] + assert model.from_wire(wire).to_wire() == wire + + +def test_unknown_additive_fields_are_tolerated(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["future_optional_field"] = True + assert ClaimRequest.from_wire(wire).capacity == 2 + + +def test_unsupported_major_version_fails_loudly(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["protocol_version"] = "3" + with pytest.raises( + UnsupportedProtocolVersion, match="supported major version is 2" + ): + ClaimRequest.from_wire(wire) + + +def test_transcript_event_count_is_an_integrity_check(): + wire = dict(_contract()["samples"]["transcript_response"]) + wire["event_count"] = 99 + with pytest.raises(ProtocolError, match="transcript contains 2 events"): + SessionTranscript.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "path", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + ("assignments", 0), + 42, + r"assignments\[0\] must be an object", + ), + ( + "register_response", + RegisterResponse, + ("disabled_definitions",), + [42], + r"disabled_definitions\[0\] must be a string", + ), + ( + "heartbeat_response", + HeartbeatResponse, + ("accepted_run_ids", 0), + None, + r"accepted_run_ids\[0\] must be a string", + ), + ( + "plan_response", + PlanResponse, + ("runs", 0), + "not-an-object", + r"runs\[0\] must be an object", + ), + ( + "result_request", + ResultRequest, + ("results", 0, "labels", 0), + 7, + r"labels\[0\] must be a string", + ), + ( + "register_request", + RegisterRequest, + ("definitions", 0, "result_kind"), + "unknown", + "result_kind must be one of", + ), + ], +) +def test_nested_wire_values_fail_with_protocol_errors( + sample, model, path, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + target = wire + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "field", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + "lease_generation", + 0, + "lease_generation must be greater than zero", + ), + ( + "claim_response", + ClaimResponse, + "event_count", + -1, + "event_count must not be negative", + ), + ( + "result_response", + ResultResponse, + "result_count", + -1, + "result_count must not be negative", + ), + ], +) +def test_server_response_counters_and_generations_are_bounded( + sample, model, field, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + if sample == "claim_response": + wire["assignments"][0][field] = value + else: + wire[field] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +def test_fixture_constants_match_the_sdk_contract(): + contract = _contract() + assert contract["protocol"] == { + "supported_major_versions": [protocol.PROTOCOL_VERSION], + "transcript_schema_version": protocol.TRANSCRIPT_SCHEMA_VERSION, + "result_schema_version": protocol.RESULT_SCHEMA_VERSION, + } + assert contract["http"] == { + "register": protocol.REGISTER_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "plan": protocol.PLAN_PATH, + "heartbeat": protocol.HEARTBEAT_PATH, + "result": protocol.RESULT_PATH, + "worker_id_header": protocol.WORKER_ID_HEADER, + "lease_generation_header": protocol.LEASE_GENERATION_HEADER, + } + assert contract["timing"] == { + "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS, + "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS, + "max_claim_wait_seconds": protocol.MAX_CLAIM_WAIT_SECONDS, + "max_attempts": protocol.MAX_ATTEMPTS, + } + assert contract["limits"] == { + "max_catalog_definitions": protocol.MAX_CATALOG_DEFINITIONS, + "max_claim_capacity": protocol.MAX_CLAIM_CAPACITY, + "max_transcript_bytes": protocol.MAX_TRANSCRIPT_BYTES, + "max_results_per_run": protocol.MAX_RESULTS_PER_RUN, + "max_eval_key_bytes": protocol.MAX_EVAL_KEY_BYTES, + "max_display_name_bytes": protocol.MAX_DISPLAY_NAME_BYTES, + "max_version_bytes": protocol.MAX_VERSION_BYTES, + "max_worker_id_bytes": protocol.MAX_WORKER_ID_BYTES, + "max_label_bytes": protocol.MAX_LABEL_BYTES, + "max_labels_per_result": protocol.MAX_LABELS_PER_RESULT, + "max_summary_bytes": protocol.MAX_SUMMARY_BYTES, + "max_reasoning_bytes": protocol.MAX_REASONING_BYTES, + "max_unit_bytes": protocol.MAX_UNIT_BYTES, + "max_display_value_bytes": protocol.MAX_DISPLAY_VALUE_BYTES, + "max_description_bytes": protocol.MAX_DESCRIPTION_BYTES, + "max_error_code_bytes": protocol.MAX_ERROR_CODE_BYTES, + "max_error_message_bytes": protocol.MAX_ERROR_MESSAGE_BYTES, + } + assert contract["errors"] == protocol.ERROR_SPECS + + +def test_session_helpers_use_the_protocol_event_vocabulary(): + session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"]) + assert session.count("tool_use") == 1 + assert session.events_of_type("agent_end")[0].payload["summary"] == "Done" diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py new file mode 100644 index 00000000..2862cfb4 --- /dev/null +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -0,0 +1,701 @@ +from __future__ import annotations + +import asyncio +import json +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimResponse, + ConditionResult, + EvalResult, + Evaluator, + EvaluatorAPIError, + HeartbeatResponse, + PlannedRun, + PlanResponse, + RegisterResponse, + ResultKind, + Score, + SessionTranscript, + WorkerConfig, + WorkerRuntime, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class FakeClient: + def __init__(self): + samples = _samples() + self.assignment = ClaimResponse.from_wire( + samples["claim_response"] + ).assignments[0] + self.session = SessionTranscript.from_wire(samples["transcript_response"]) + self.register_requests = [] + self.claim_requests = [] + self.plans = [] + self.submissions = [] + self.heartbeats = [] + + def register(self, request): + self.register_requests.append(request) + return RegisterResponse.from_wire(_samples()["register_response"]) + + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=(self.assignment,)) + + def transcript(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return self.session + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned" if request.selected else "skipped", + runs=tuple( + PlannedRun(f"run-{item.eval_key}", item.eval_key, item.eval_version) + for item in request.selected + ), + ) + + def submit_result(self, run_id, request): + self.submissions.append((run_id, request)) + + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=tuple(item.evaluation_run_id for item in request.runs), + ) + + +def _runtime(evaluator, client): + return WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=2, + claim_wait_seconds=1, + ), + client=client, + ) + + +def test_condition_failures_are_isolated_and_plan_is_declared_first(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("not_applicable", version="1", when=lambda session: False) + def not_applicable(session): + return EvalResult(score=Score(1)) + + def broken_condition(session): + raise RuntimeError("condition exploded") + + @evaluator.eval("broken_condition", version="1", when=broken_condition) + def never_runs(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + assert len(client.plans) == 1 + assert [item.eval_key for item in client.plans[0].selected] == ["selected"] + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("not_applicable", "condition_false"), + ("broken_condition", "condition_error"), + } + assert [run_id for run_id, _ in client.submissions] == ["run-selected"] + + +def test_condition_can_supply_a_stable_skip_reason(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval( + "retrieval_only", + version="1", + when=lambda session: ConditionResult(False, "no_retrieval_events"), + ) + def retrieval_only(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans[0].skipped[0].reason_code == "no_retrieval_events" + + +def test_one_eval_failure_does_not_block_another_result(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("fails", version="1") + def fails(session): + raise RuntimeError("secret details should be bounded") + + @evaluator.eval("succeeds", version="1") + async def succeeds(session): + await asyncio.sleep(0) + return EvalResult(score=Score(0.8), summary="good") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + by_run = {run_id: request for run_id, request in client.submissions} + assert by_run["run-fails"].status.value == "failed" + assert by_run["run-fails"].error_code == "eval_error" + assert by_run["run-fails"].results == () + assert by_run["run-succeeds"].status.value == "succeeded" + assert by_run["run-succeeds"].results[0].result_kind == ResultKind.SCORE + + +def test_timeout_is_submitted_as_a_terminal_run(): + evaluator = Evaluator(name="test", version="1") + cancelled = [] + + @evaluator.eval( + "slow", + version="1", + timeout_seconds=0.01, + on_cancel=lambda session: cancelled.append(session.session_revision_id), + ) + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert cancelled == [client.assignment.session_revision_id] + + +def test_lost_lease_cancels_local_execution(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1") + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + class LeaseLostClient(FakeClient): + def heartbeat(self, request): + raise EvaluatorAPIError( + status=409, + code="lease_lost", + message="gone", + retryable=False, + ) + + client = LeaseLostClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + with pytest.raises(asyncio.CancelledError): + asyncio.run(runtime.process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_partial_heartbeat_acceptance_cancels_only_the_fenced_run(): + evaluator = Evaluator(name="test", version="1") + + class PartialHeartbeatClient(FakeClient): + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=(request.runs[0].evaluation_run_id,), + ) + + client = PartialHeartbeatClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + first = asyncio.create_task(asyncio.sleep(60)) + second = asyncio.create_task(asyncio.sleep(60)) + heartbeat = asyncio.create_task( + runtime._heartbeat( + client.assignment, {"run-first": first, "run-second": second} + ) + ) + while not client.heartbeats: + await asyncio.sleep(0.001) + for _ in range(100): + if second.done(): + break + await asyncio.sleep(0.001) + assert first.done() is False + assert second.cancelled() is True + heartbeat.cancel() + first.cancel() + await asyncio.gather(first, second, heartbeat, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_transcript_revision_must_match_the_claimed_assignment(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + client.session = SessionTranscript.from_wire( + { + **_samples()["transcript_response"], + "session_revision_id": "different-revision", + } + ) + + with pytest.raises(RuntimeError, match="revision does not match"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans == [] + + +def test_server_cannot_add_a_run_when_every_eval_was_skipped(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + class UnexpectedRunClient(FakeClient): + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-injected", "skipped", "1"),), + ) + + client = UnexpectedRunClient() + with pytest.raises(RuntimeError, match="unrequested evaluation run"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_server_plan_must_match_assignment_and_include_each_selected_eval(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1") + def selected(session): + return EvalResult(score=Score(1)) + + class WrongAssignmentClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id="another-assignment", + assignment_status="planned", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_assignment = WrongAssignmentClient() + with pytest.raises(RuntimeError, match="different assignment"): + asyncio.run( + _runtime(evaluator, wrong_assignment).process_assignment( + wrong_assignment.assignment + ) + ) + + class WrongStatusClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_status = WrongStatusClient() + with pytest.raises(RuntimeError, match="inconsistent assignment status"): + asyncio.run( + _runtime(evaluator, wrong_status).process_assignment( + wrong_status.assignment + ) + ) + + class OmittedRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + ) + + omitted = OmittedRunClient() + with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): + asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + + +def test_server_plan_rejects_duplicate_run_ids(): + evaluator = Evaluator(name="test", version="1") + evaluator.eval("first", version="1")(lambda session: EvalResult(score=Score(1))) + evaluator.eval("second", version="1")(lambda session: EvalResult(score=Score(1))) + + class DuplicateRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun("same-run", "first", "1"), + PlannedRun("same-run", "second", "1"), + ), + ) + + client = DuplicateRunClient() + with pytest.raises(RuntimeError, match="duplicate evaluation run id"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_register_advertises_the_deterministic_catalog(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("quality", version="7") + def quality(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + asyncio.run(runtime.register()) + request = client.register_requests[0] + assert request.catalog_revision == evaluator.catalog_revision + assert request.definitions[0].eval_version == "7" + assert runtime._heartbeat_interval == 30 + + +def test_runtime_readiness_tracks_registration_contact_and_shutdown(monkeypatch): + evaluator = Evaluator(name="test", version="1") + runtime = _runtime(evaluator, FakeClient()) + + assert runtime.is_ready() is False + assert runtime.metrics() == {} + + asyncio.run(runtime.register()) + assert runtime.is_ready() is True + assert runtime.metrics() == {"registration_success": 1} + + last_contact = runtime._last_server_contact + assert last_contact is not None + monkeypatch.setattr(time, "monotonic", lambda: last_contact + 121) + assert runtime.is_ready() is False + + monkeypatch.setattr(time, "monotonic", lambda: last_contact) + runtime.stop() + assert runtime.is_ready() is False + + +def test_runtime_metrics_count_claims_conditions_and_outcomes(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + runtime = _runtime(evaluator, FakeClient()) + + async def exercise(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(exercise()) == 1 + assert runtime.metrics() == { + "assignments_claimed": 1, + "conditions_selected": 1, + "conditions_skipped": 1, + "registration_success": 1, + "runs_succeeded": 1, + } + + +def test_runtime_metrics_count_registration_failure(): + evaluator = Evaluator(name="test", version="1") + + class BrokenClient(FakeClient): + def register(self, request): + raise EvaluatorAPIError( + status=503, + code="unavailable", + message="try later", + retryable=True, + ) + + runtime = _runtime(evaluator, BrokenClient()) + with pytest.raises(EvaluatorAPIError): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_invalid_registration_response_does_not_make_runtime_ready(): + evaluator = Evaluator(name="test", version="1") + + class InvalidTimingClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=120, + lease_duration_seconds=120, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, InvalidTimingClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_lost_claim_response_waits_out_the_lease_before_claiming_again(): + evaluator = Evaluator(name="test", version="1") + + class LostResponseClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=None, + code="transport_error", + message="response lost", + retryable=True, + ) + + runtime = _runtime(evaluator, LostResponseClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [120.0] + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +def test_nonretryable_claim_failure_stops_the_worker(): + evaluator = Evaluator(name="test", version="1") + + class RejectedClaimClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=409, + code="catalog_mismatch", + message="register again with the current catalog", + retryable=False, + ) + + runtime = _runtime(evaluator, RejectedClaimClient()) + with pytest.raises(EvaluatorAPIError, match="catalog_mismatch"): + asyncio.run(runtime.run_forever()) + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +@pytest.mark.parametrize( + ("assignments", "message"), + [ + (lambda item: (item, item), "duplicate assignments"), + ( + lambda item: tuple( + replace(item, assignment_id=f"assignment-{index}") for index in range(3) + ), + "more assignments than requested", + ), + ], +) +def test_claim_response_cannot_exceed_capacity_or_repeat_work(assignments, message): + evaluator = Evaluator(name="test", version="1") + + class InvalidClaimClient(FakeClient): + def claim(self, request): + return ClaimResponse(assignments=assignments(self.assignment)) + + runtime = _runtime(evaluator, InvalidClaimClient()) + with pytest.raises(RuntimeError, match=message): + asyncio.run(runtime.run_once()) + assert runtime.metrics() == {} + + +def test_register_applies_server_claim_limit_and_disabled_definitions(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("disabled", version="1") + def disabled(session): + raise AssertionError("disabled eval must not run") + + class RestrictedClient(FakeClient): + def register(self, request): + self.register_requests.append(request) + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=10, + lease_duration_seconds=120, + claim_limit=1, + disabled_definitions=("disabled",), + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + client = RestrictedClient() + runtime = _runtime(evaluator, client) + + async def exercise(): + await runtime.register() + await runtime.run_once() + + asyncio.run(exercise()) + assert runtime._claim_limit == 1 + assert client.claim_requests[0].capacity == 1 + assert client.plans[0].selected == () + assert client.plans[0].skipped[0].reason_code == "disabled_by_server" + + +def test_worker_config_requires_dedicated_credentials(monkeypatch): + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_URL", raising=False) + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_TOKEN", raising=False) + with pytest.raises(ValueError, match="URL is required"): + WorkerConfig.from_env() + + +def test_worker_config_keeps_long_poll_inside_the_http_timeout(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", "20") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", "20") + with pytest.raises(ValueError, match="must exceed"): + WorkerConfig.from_env() + + +def test_worker_config_rejects_header_control_characters(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_WORKER_ID", "worker\nforged") + with pytest.raises(ValueError, match="control characters"): + WorkerConfig.from_env() + + +def test_graceful_drain_cancels_work_after_the_configured_deadline(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + drain_timeout_seconds=1, + ), + client=client, + ) + cancelled = False + + async def exercise(): + nonlocal cancelled + + async def active_work(): + nonlocal cancelled + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled = True + raise + + task = asyncio.create_task(active_work()) + runtime._active.add(task) + await asyncio.sleep(0) + runtime.config = WorkerConfig( + server_url=runtime.config.server_url, + credential=runtime.config.credential, + worker_id=runtime.config.worker_id, + drain_timeout_seconds=0, + ) + await runtime.drain() + + asyncio.run(exercise()) + assert cancelled is True + assert runtime._active == set() + + +def test_stop_interrupts_capacity_wait_and_enters_drain(): + runtime = _runtime(Evaluator(name="test", version="1"), FakeClient()) + + async def exercise(): + blocker = asyncio.Event() + work = asyncio.create_task(blocker.wait()) + runtime._active.add(work) + await asyncio.sleep(0) + + runtime.stop() + await asyncio.wait_for(runtime._wait_for_progress(), timeout=0.1) + + assert work.done() is False + work.cancel() + await asyncio.gather(work, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_eval_execution_respects_process_concurrency(): + evaluator = Evaluator(name="test", version="1") + active = 0 + peak = 0 + + async def measured(session): + nonlocal active, peak + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) + active -= 1 + return EvalResult(score=Score(1)) + + evaluator.eval("first", version="1")(measured) + evaluator.eval("second", version="1")(measured) + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=1, + ), + client=client, + ) + asyncio.run(runtime.process_assignment(client.assignment)) + assert peak == 1 diff --git a/sdk/python/tests/test_zero_dependencies.py b/sdk/python/tests/test_zero_dependencies.py index a27fd75f..95be7b72 100644 --- a/sdk/python/tests/test_zero_dependencies.py +++ b/sdk/python/tests/test_zero_dependencies.py @@ -300,6 +300,23 @@ def test_importing_the_package_loads_no_framework(): ) +def test_importing_the_package_does_not_load_the_evaluator_runtime(): + """Telemetry-only users do not pay for the separate worker surface.""" + import json + import subprocess + + probe = ( + "import json, sys; import failproofai_sdk; " + "print(json.dumps(sorted(m for m in sys.modules " + "if m.startswith('failproofai_sdk.evaluator'))))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, cwd=str(ROOT), timeout=60 + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip()) == [] + + def test_the_adapter_registry_holds_strings_not_modules(): """`_REGISTRY` maps a name to a dotted path; importing it here would defeat it.""" from failproofai_sdk.integrations import _REGISTRY From 0a68c7ac7183941bf5b8c36f42ec886eb1b05e87 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 17:31:11 +0530 Subject: [PATCH 3/4] fix(evaluator): resume only unfinished replayed runs --- sdk/python/failproofai_sdk/evaluator/protocol.py | 5 +++++ sdk/python/failproofai_sdk/evaluator/runtime.py | 2 +- .../tests/fixtures/evaluator_v2/contract.json | 1 + sdk/python/tests/test_evaluator_runtime.py | 15 ++++++++++++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index 09adf8be..beb2391a 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -452,17 +452,22 @@ class PlanResponse(WireModel): assignment_id: str assignment_status: str runs: tuple[PlannedRun, ...] + idempotent_replay: bool = False protocol_version: str = PROTOCOL_VERSION @classmethod def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay", False) + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") return cls( assignment_id=_string(data, "assignment_id"), assignment_status=_string(data, "assignment_status"), runs=tuple( PlannedRun.from_wire(item) for item in _object_list(data, "runs") ), + idempotent_replay=replay, ) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 8a21402c..600713cc 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -334,7 +334,7 @@ async def process_assignment(self, assignment: Assignment) -> None: if definition is None: raise RuntimeError("server returned an unrequested evaluation run") run_definitions.append((run.evaluation_run_id, definition)) - if definitions: + if definitions and not plan.idempotent_replay: raise RuntimeError("server omitted a selected evaluation run") tasks = { diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index aa0cf275..fdc1a0cf 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -150,6 +150,7 @@ "protocol_version": "2", "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", "assignment_status": "planned", + "idempotent_replay": false, "runs": [ { "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index 2862cfb4..bc770e5c 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -285,7 +285,7 @@ def plan(self, assignment_id, request): assert client.submissions == [] -def test_server_plan_must_match_assignment_and_include_each_selected_eval(): +def test_server_plan_must_match_assignment_and_include_each_new_selected_eval(): evaluator = Evaluator(name="test", version="1") @evaluator.eval("selected", version="1") @@ -336,6 +336,19 @@ def plan(self, assignment_id, request): with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + class ReplayedPlanClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + idempotent_replay=True, + ) + + replayed = ReplayedPlanClient() + asyncio.run(_runtime(evaluator, replayed).process_assignment(replayed.assignment)) + assert replayed.submissions == [] + def test_server_plan_rejects_duplicate_run_ids(): evaluator = Evaluator(name="test", version="1") From dfdb08be2e940a3e4c02239a23d2b8625d0c44ef Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 18:59:03 +0530 Subject: [PATCH 4/4] feat(sdk): run hosted evaluator definitions --- sdk/python/examples/evaluator_worker.py | 10 + .../failproofai_sdk/evaluator/__init__.py | 16 ++ .../failproofai_sdk/evaluator/client.py | 33 ++- .../failproofai_sdk/evaluator/protocol.py | 91 ++++++- .../failproofai_sdk/evaluator/runtime.py | 236 ++++++++++++++---- .../failproofai_sdk/evaluator/source.py | 170 +++++++++++++ .../tests/fixtures/evaluator_v2/contract.json | 30 ++- sdk/python/tests/test_evaluator_client.py | 32 +++ sdk/python/tests/test_evaluator_protocol.py | 7 +- sdk/python/tests/test_evaluator_runtime.py | 133 ++++++++++ sdk/python/tests/test_evaluator_source.py | 60 +++++ 11 files changed, 755 insertions(+), 63 deletions(-) create mode 100644 sdk/python/failproofai_sdk/evaluator/source.py create mode 100644 sdk/python/tests/test_evaluator_source.py diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py index a0cea74b..912b61e5 100644 --- a/sdk/python/examples/evaluator_worker.py +++ b/sdk/python/examples/evaluator_worker.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import ipaddress import json import os from urllib.parse import urlsplit @@ -73,6 +74,15 @@ def _call_judge(question, answer): parsed = urlsplit(url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback: + raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback") token = os.environ.get("EXAMPLE_JUDGE_TOKEN") body = json.dumps({"question": question, "answer": answer}).encode("utf-8") headers = {"Content-Type": "application/json", "Accept": "application/json"} diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py index dff0cad6..a94155ce 100644 --- a/sdk/python/failproofai_sdk/evaluator/__init__.py +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -16,11 +16,13 @@ from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient from failproofai_sdk.evaluator.protocol import ( Assignment, + AssignmentDefinition, CatalogDefinition, ClaimRequest, ClaimResponse, ErrorResponse, EvalSelection, + ExecutionMode, EvaluatorKind, HeartbeatRequest, HeartbeatResponse, @@ -28,6 +30,7 @@ PlannedRun, PlanRequest, PlanResponse, + DefinitionsResponse, ProtocolError, RegisterRequest, RegisterResponse, @@ -43,10 +46,17 @@ UnsupportedProtocolVersion, ) from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime +from failproofai_sdk.evaluator.source import ( + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) __all__ = [ "Assertion", "Assignment", + "AssignmentDefinition", "CatalogDefinition", "ClaimRequest", "ClaimResponse", @@ -55,6 +65,7 @@ "EvalDefinition", "EvalResult", "EvalSelection", + "ExecutionMode", "Evaluator", "EvaluatorAPIError", "EvaluatorClient", @@ -65,6 +76,7 @@ "Metric", "PlanRequest", "PlanResponse", + "DefinitionsResponse", "PlannedRun", "ProtocolError", "RegisterRequest", @@ -82,4 +94,8 @@ "UnsupportedProtocolVersion", "WorkerConfig", "WorkerRuntime", + "UnsafeEvaluatorSource", + "compile_condition", + "compile_evaluator", + "source_checksum", ] diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py index 60fa32c2..fc3c28ce 100644 --- a/sdk/python/failproofai_sdk/evaluator/client.py +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ipaddress import json import random import time @@ -13,6 +14,7 @@ from failproofai_sdk.evaluator.protocol import ( CLAIM_PATH, + DEFINITIONS_PATH, HEARTBEAT_PATH, LEASE_GENERATION_HEADER, MAX_TRANSCRIPT_BYTES, @@ -21,6 +23,7 @@ RESULT_PATH, WORKER_ID_HEADER, Assignment, + DefinitionsResponse, ClaimRequest, ClaimResponse, ErrorResponse, @@ -67,7 +70,11 @@ def __init__( class EvaluatorClient: - """Direct server client; evaluator traffic never passes through the dashboard.""" + """Client for the public Evaluator v2 machine API. + + Hosted workers normally use the FailproofAI dashboard origin. Its ``/v1`` + passthrough forwards this worker's bearer credential to the private server. + """ def __init__( self, @@ -76,12 +83,22 @@ def __init__( credential: str, timeout_seconds: float = 30, max_retries: int = 3, + allow_insecure_http: bool = False, opener: Callable[..., Any] | None = None, sleeper: Callable[[float], None] = time.sleep, ) -> None: parsed = urlsplit(base_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("base_url must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback and not allow_insecure_http: + raise ValueError("base_url must use https unless it targets loopback") if not credential or not credential.strip(): raise ValueError("credential must not be empty") if any( @@ -130,6 +147,20 @@ def transcript( ) ) + def definitions( + self, assignment: Assignment, *, worker_id: str + ) -> DefinitionsResponse: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + path = assignment.definitions_url or DEFINITIONS_PATH.format( + assignment_id=assignment.assignment_id + ) + return DefinitionsResponse.from_wire( + self._json("GET", path, None, retry=True, headers=headers) + ) + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: return PlanResponse.from_wire( self._json( diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index beb2391a..e8fa156c 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -15,6 +15,7 @@ REGISTER_PATH = "/v1/evaluator/workers/register" CLAIM_PATH = "/v1/evaluator/assignments/claim" TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +DEFINITIONS_PATH = "/v1/evaluator/assignments/{assignment_id}/definitions" PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" @@ -91,6 +92,11 @@ class ResultKind(str, Enum): ASSERTION = "assertion" +class ExecutionMode(str, Enum): + LOCAL = "local" + PYTHON = "python" + + class TerminalRunStatus(str, Enum): SUCCEEDED = "succeeded" FAILED = "failed" @@ -287,6 +293,7 @@ class Assignment(WireModel): trigger_reason: str event_count: int transcript_url: str + definitions_url: str = "" @classmethod def from_wire(cls, data: Mapping[str, Any]) -> Assignment: @@ -301,6 +308,65 @@ def from_wire(cls, data: Mapping[str, Any]) -> Assignment: trigger_reason=_string(data, "trigger_reason"), event_count=_nonnegative_integer(data, "event_count"), transcript_url=_string(data, "transcript_url"), + definitions_url=str(data.get("definitions_url") or ""), + ) + + +@dataclass(frozen=True) +class AssignmentDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + execution_mode: ExecutionMode = ExecutionMode.LOCAL + condition_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> AssignmentDefinition: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + execution_mode=_enum( + ExecutionMode, + {"execution_mode": data.get("execution_mode") or "local"}, + "execution_mode", + ), + condition_source=_optional_string(data, "condition_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class DefinitionsResponse(WireModel): + assignment_id: str + catalog_revision: str + definitions: tuple[AssignmentDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> DefinitionsResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + catalog_revision=_string(data, "catalog_revision"), + definitions=tuple( + AssignmentDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), ) @@ -437,13 +503,32 @@ class PlannedRun(WireModel): evaluation_run_id: str eval_key: str eval_version: str + execution_mode: ExecutionMode = ExecutionMode.LOCAL + evaluator_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None @classmethod def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") return cls( - _string(data, "evaluation_run_id"), - _string(data, "eval_key"), - _string(data, "eval_version"), + evaluation_run_id=_string(data, "evaluation_run_id"), + eval_key=_string(data, "eval_key"), + eval_version=_string(data, "eval_version"), + execution_mode=_enum( + ExecutionMode, + {"execution_mode": data.get("execution_mode") or "local"}, + "execution_mode", + ), + evaluator_source=_optional_string(data, "evaluator_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, ) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 600713cc..f96d205f 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import concurrent.futures import inspect import logging import os @@ -27,8 +28,10 @@ MAX_CLAIM_WAIT_SECONDS, MAX_WORKER_ID_BYTES, Assignment, + AssignmentDefinition, ClaimRequest, EvalSelection, + ExecutionMode, HeartbeatRequest, HeartbeatRun, PlanRequest, @@ -37,6 +40,11 @@ SkippedEval, TerminalRunStatus, ) +from failproofai_sdk.evaluator.source import ( + compile_condition, + compile_evaluator, + source_checksum, +) logger = logging.getLogger("failproofai_sdk.evaluator") @@ -62,6 +70,18 @@ def _positive_int(name: str, default: int) -> int: return value +def _boolean(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + normalized = raw.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + @dataclass(frozen=True) class WorkerConfig: server_url: str @@ -71,6 +91,7 @@ class WorkerConfig: claim_wait_seconds: int = 20 request_timeout_seconds: int = 30 drain_timeout_seconds: int = 60 + allow_insecure_http: bool = False @classmethod def from_env(cls) -> WorkerConfig: @@ -105,6 +126,9 @@ def from_env(cls) -> WorkerConfig: drain_timeout_seconds=_positive_int( "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 ), + allow_insecure_http=_boolean( + "FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP" + ), ) if config.max_concurrency > MAX_CLAIM_CAPACITY: raise ValueError( @@ -137,6 +161,7 @@ def __init__( base_url=config.server_url, credential=config.credential, timeout_seconds=config.request_timeout_seconds, + allow_insecure_http=config.allow_insecure_http, ) self._stopping = asyncio.Event() self._active: set[asyncio.Task[None]] = set() @@ -145,6 +170,10 @@ def __init__( self._lease_duration = 120 self._disabled_definitions: set[str] = set() self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + self._eval_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=config.max_concurrency, + thread_name_prefix="failproof-eval", + ) self._registered = False self._last_server_contact: float | None = None self._metric_lock = threading.Lock() @@ -183,44 +212,51 @@ async def register(self) -> None: async def run_forever(self) -> None: await self.register() - while not self._stopping.is_set(): - self._reap_finished() - capacity = self._claim_limit - len(self._active) - if capacity <= 0: - await self._wait_for_progress() - continue - try: - response = await self._call_client( - self.client.claim, - ClaimRequest( - worker_id=self.config.worker_id, - catalog_revision=self.evaluator.catalog_revision, - capacity=capacity, - wait_seconds=self.config.claim_wait_seconds, - ), - ) - except EvaluatorAPIError as error: - self._increment("claim_failures") - logger.warning( - "evaluator claim failed", - extra={"code": error.code, "retryable": error.retryable}, - ) - if not error.retryable: - raise - # With a transport error the server may have committed the - # lease while its response was lost. Waiting out that lease is - # what prevents a blind second claim from exceeding capacity. - await self._wait_or_stop( - float(self._lease_duration) if error.status is None else 1.0 + retry_delay = 1.0 + try: + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + delay = ( + float(self._lease_duration) + if error.status is None + else retry_delay + ) + await self._wait_or_stop(delay) + retry_delay = min(retry_delay * 2.0, 30.0) + continue + retry_delay = 1.0 + assignments = self._validated_assignments( + response.assignments, capacity ) - continue - assignments = self._validated_assignments(response.assignments, capacity) - for assignment in assignments: - task = asyncio.create_task(self.process_assignment(assignment)) - self._active.add(task) - self._increment("assignments_claimed", len(assignments)) - - await self.drain() + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + finally: + await self.drain() + self._eval_executor.shutdown(wait=False, cancel_futures=True) async def run_once(self) -> int: """Claim once and finish the returned assignments; useful for jobs/tests.""" @@ -267,18 +303,37 @@ async def process_assignment(self, assignment: Assignment) -> None: if session.session_revision_id != assignment.session_revision_id: raise RuntimeError("transcript session revision does not match assignment") - selected: list[EvalDefinition] = [] + descriptors = await self._assignment_definitions(assignment) + selected: list[tuple[AssignmentDefinition, EvalDefinition | None]] = [] skipped: list[SkippedEval] = [] - for definition in self.evaluator.definitions: - if definition.eval_key in self._disabled_definitions: - skipped.append(self._skipped(definition, "disabled_by_server")) + local_definitions = { + (item.eval_key, item.eval_version): item + for item in self.evaluator.definitions + } + for descriptor in descriptors: + local = local_definitions.get( + (descriptor.eval_key, descriptor.eval_version) + ) + if descriptor.execution_mode is ExecutionMode.LOCAL and local is None: + raise RuntimeError("server requested a definition absent from this worker") + if descriptor.eval_key in self._disabled_definitions: + skipped.append(self._skipped_descriptor(descriptor, "disabled_by_server")) self._increment("conditions_skipped") continue - if definition.condition is None: - selected.append(definition) + condition_function = ( + local.condition + if local is not None + else ( + compile_condition(descriptor.condition_source) + if descriptor.condition_source + else None + ) + ) + if condition_function is None: + selected.append((descriptor, local)) continue try: - condition = await self._invoke(definition.condition, session) + condition = await self._invoke(condition_function, session) if isinstance(condition, ConditionResult): applicable = condition.applicable reason_code = condition.reason_code @@ -295,14 +350,14 @@ async def process_assignment(self, assignment: Assignment) -> None: "error_type": type(error).__name__, }, ) - skipped.append(self._skipped(definition, "condition_error")) + skipped.append(self._skipped_descriptor(descriptor, "condition_error")) self._increment("conditions_skipped") continue if applicable: - selected.append(definition) + selected.append((descriptor, local)) self._increment("conditions_selected") else: - skipped.append(self._skipped(definition, reason_code)) + skipped.append(self._skipped_descriptor(descriptor, reason_code)) self._increment("conditions_skipped") plan = await self._call_client( @@ -312,7 +367,8 @@ async def process_assignment(self, assignment: Assignment) -> None: worker_id=self.config.worker_id, lease_generation=assignment.lease_generation, selected=tuple( - EvalSelection(item.eval_key, item.eval_version) for item in selected + EvalSelection(item.eval_key, item.eval_version) + for item, _local in selected ), skipped=tuple(skipped), ), @@ -323,16 +379,50 @@ async def process_assignment(self, assignment: Assignment) -> None: if plan.assignment_status != expected_status: raise RuntimeError("server returned an inconsistent assignment status") - definitions = {(item.eval_key, item.eval_version): item for item in selected} + definitions = { + (item.eval_key, item.eval_version): (item, local) + for item, local in selected + } run_definitions: list[tuple[str, EvalDefinition]] = [] run_ids: set[str] = set() for run in plan.runs: if run.evaluation_run_id in run_ids: raise RuntimeError("server returned a duplicate evaluation run id") run_ids.add(run.evaluation_run_id) - definition = definitions.pop((run.eval_key, run.eval_version), None) - if definition is None: + selected_definition = definitions.pop( + (run.eval_key, run.eval_version), None + ) + if selected_definition is None: raise RuntimeError("server returned an unrequested evaluation run") + descriptor, local = selected_definition + if run.execution_mode is not descriptor.execution_mode: + raise RuntimeError("server changed the evaluation execution mode") + if run.execution_mode is ExecutionMode.LOCAL: + if local is None: + raise RuntimeError("local evaluation definition is unavailable") + definition = local + else: + if not run.evaluator_source or not run.source_checksum: + raise RuntimeError("server omitted managed evaluation source") + expected = source_checksum( + descriptor.condition_source, run.evaluator_source + ) + if expected != run.source_checksum or ( + descriptor.source_checksum + and descriptor.source_checksum != run.source_checksum + ): + raise RuntimeError("managed evaluation source checksum mismatch") + definition = EvalDefinition( + eval_key=descriptor.eval_key, + display_name=descriptor.display_name, + eval_version=descriptor.eval_version, + result_kind=descriptor.result_kind, + labels=descriptor.labels, + function=compile_evaluator(run.evaluator_source), + condition=None, + on_cancel=None, + timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, + ) run_definitions.append((run.evaluation_run_id, definition)) if definitions and not plan.idempotent_replay: raise RuntimeError("server omitted a selected evaluation run") @@ -479,12 +569,21 @@ async def _heartbeat( }, ) self._increment("heartbeat_failures") + except Exception as error: # noqa: BLE001 - keep lease renewal alive + logger.warning( + "evaluator heartbeat error", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + self._increment("heartbeat_failures") - @staticmethod - async def _invoke(function, session): + async def _invoke(self, function, session): if inspect.iscoroutinefunction(function): return await function(session) - result = await asyncio.to_thread(function, session) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(self._eval_executor, function, session) if inspect.isawaitable(result): return await result return result @@ -493,6 +592,35 @@ async def _invoke(function, session): def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: return SkippedEval(definition.eval_key, definition.eval_version, reason) + @staticmethod + def _skipped_descriptor( + definition: AssignmentDefinition, reason: str + ) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + async def _assignment_definitions( + self, assignment: Assignment + ) -> tuple[AssignmentDefinition, ...]: + if assignment.definitions_url: + response = await self._call_client( + self.client.definitions, + assignment, + worker_id=self.config.worker_id, + ) + if response.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned definitions for another assignment") + return response.definitions + return tuple( + AssignmentDefinition( + eval_key=item.eval_key, + display_name=item.display_name, + eval_version=item.eval_version, + result_kind=item.result_kind, + labels=item.labels, + ) + for item in self.evaluator.definitions + ) + def _reap_finished(self) -> None: done = {task for task in self._active if task.done()} self._active.difference_update(done) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py new file mode 100644 index 00000000..b3c7da9f --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -0,0 +1,170 @@ +"""Restricted deterministic expression compiler for server-authored evaluations. + +The managed worker never executes a module, statements, imports, or ambient +builtins from tenant-authored source. Definitions are single Python expressions +evaluated with a small constructor/helper surface and the immutable transcript +bound as ``session``. +""" + +from __future__ import annotations + +import ast +import hashlib +from collections.abc import Callable +from typing import Any + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalResult, + Metric, + Score, +) + +MAX_CONDITION_SOURCE_BYTES = 16 * 1024 +MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024 + +_ALLOWED_NODES = ( + ast.Expression, + ast.BoolOp, + ast.BinOp, + ast.UnaryOp, + ast.IfExp, + ast.Dict, + ast.Set, + ast.List, + ast.Tuple, + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, + ast.comprehension, + ast.Compare, + ast.Call, + ast.FormattedValue, + ast.JoinedStr, + ast.Constant, + ast.Name, + ast.Load, + ast.Store, + ast.Attribute, + ast.Subscript, + ast.Slice, + ast.keyword, + ast.And, + ast.Or, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.USub, + ast.UAdd, + ast.Not, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, +) + +_SAFE_GLOBALS = { + "__builtins__": {}, + "Assertion": Assertion, + "ConditionResult": ConditionResult, + "EvalResult": EvalResult, + "Metric": Metric, + "Score": Score, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + "enumerate": enumerate, + "float": float, + "int": int, + "len": len, + "list": list, + "max": max, + "min": min, + "range": range, + "round": round, + "set": set, + "sorted": sorted, + "str": str, + "sum": sum, + "tuple": tuple, +} + + +class UnsafeEvaluatorSource(ValueError): + """Raised before any disallowed server-authored source can execute.""" + + +def source_checksum(condition_source: str | None, evaluator_source: str) -> str: + payload = (condition_source or "").encode("utf-8") + b"\0" + evaluator_source.encode( + "utf-8" + ) + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _compile(source: str, *, field_name: str, maximum: int) -> Any: + if not isinstance(source, str) or not source.strip(): + raise UnsafeEvaluatorSource(f"{field_name} must not be empty") + if len(source.encode("utf-8")) > maximum: + raise UnsafeEvaluatorSource(f"{field_name} exceeds {maximum} bytes") + try: + tree = ast.parse(source, mode="eval") + except SyntaxError as error: + raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + for node in ast.walk(tree): + if not isinstance(node, _ALLOWED_NODES): + raise UnsafeEvaluatorSource( + f"{field_name} contains disallowed syntax: {type(node).__name__}" + ) + if isinstance(node, ast.Attribute) and node.attr.startswith("_"): + raise UnsafeEvaluatorSource( + f"{field_name} may not access private or dunder attributes" + ) + if isinstance(node, ast.Name) and node.id.startswith("_"): + raise UnsafeEvaluatorSource(f"{field_name} may not access private names") + return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) + + +def compile_condition(source: str) -> Callable[[Any], bool | ConditionResult]: + code = _compile( + source, + field_name="condition_source", + maximum=MAX_CONDITION_SOURCE_BYTES, + ) + + def condition(session: Any) -> bool | ConditionResult: + value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + if not isinstance(value, (bool, ConditionResult)): + raise TypeError("condition_source must return bool or ConditionResult") + return value + + return condition + + +def compile_evaluator(source: str) -> Callable[[Any], EvalResult]: + code = _compile( + source, + field_name="evaluator_source", + maximum=MAX_EVALUATOR_SOURCE_BYTES, + ) + + def evaluate(session: Any) -> EvalResult: + value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + if not isinstance(value, EvalResult): + raise TypeError("evaluator_source must return EvalResult") + return value + + return evaluate diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index fdc1a0cf..5998f535 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -1,5 +1,5 @@ { - "fixture_revision": "evaluator-v2-2026-08-28.2", + "fixture_revision": "evaluator-v2-2026-08-28.3", "protocol": { "supported_major_versions": ["2"], "transcript_schema_version": "2", @@ -9,6 +9,7 @@ "register": "/v1/evaluator/workers/register", "claim": "/v1/evaluator/assignments/claim", "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "definitions": "/v1/evaluator/assignments/{assignment_id}/definitions", "plan": "/v1/evaluator/assignments/{assignment_id}/plan", "heartbeat": "/v1/evaluator/runs/heartbeat", "result": "/v1/evaluator/runs/{evaluation_run_id}/result", @@ -102,7 +103,26 @@ "environment": "production", "trigger_reason": "agent_end", "event_count": 42, - "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript" + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript", + "definitions_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/definitions" + } + ] + }, + "definitions_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"], + "execution_mode": "python", + "condition_source": null, + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 } ] }, @@ -155,7 +175,11 @@ { "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", "eval_key": "tool_efficiency", - "eval_version": "1.2.0" + "eval_version": "1.2.0", + "execution_mode": "python", + "evaluator_source": "EvalResult(score=Score(1.0))", + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 } ] }, diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py index 7513e306..d74c60dc 100644 --- a/sdk/python/tests/test_evaluator_client.py +++ b/sdk/python/tests/test_evaluator_client.py @@ -163,6 +163,28 @@ def opener(request, timeout): assert captured.get_header("X-failproofai-lease-generation") == "3" +def test_definitions_use_the_server_supplied_path_and_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["definitions_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + response = client.definitions(assignment, worker_id="worker-7") + + assert response.assignment_id == assignment.assignment_id + assert response.definitions[0].execution_mode.value == "python" + assert captured.full_url.endswith(assignment.definitions_url) + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + def test_constructor_rejects_unsafe_or_incomplete_configuration(): with pytest.raises(ValueError, match="absolute"): EvaluatorClient(base_url="localhost:8080", credential="secret") @@ -172,6 +194,16 @@ def test_constructor_rejects_unsafe_or_incomplete_configuration(): EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") +def test_private_cluster_http_requires_an_explicit_opt_in(): + with pytest.raises(ValueError, match="must use https"): + EvaluatorClient(base_url="http://server:8080", credential="secret") + EvaluatorClient( + base_url="http://server:8080", + credential="secret", + allow_insecure_http=True, + ) + + def test_protocol_redirect_does_not_forward_the_bearer_credential(): exfiltration_attempts = [] diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py index a7fd18ee..e9c2cc89 100644 --- a/sdk/python/tests/test_evaluator_protocol.py +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -9,6 +9,7 @@ from failproofai_sdk.evaluator import ( ClaimRequest, ClaimResponse, + DefinitionsResponse, ErrorResponse, HeartbeatRequest, HeartbeatResponse, @@ -38,6 +39,7 @@ def _contract(): ("register_response", RegisterResponse), ("claim_request", ClaimRequest), ("claim_response", ClaimResponse), + ("definitions_response", DefinitionsResponse), ("transcript_response", SessionTranscript), ("plan_request", PlanRequest), ("plan_response", PlanResponse), @@ -181,8 +183,9 @@ def test_fixture_constants_match_the_sdk_contract(): } assert contract["http"] == { "register": protocol.REGISTER_PATH, - "claim": protocol.CLAIM_PATH, - "transcript": protocol.TRANSCRIPT_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "definitions": protocol.DEFINITIONS_PATH, "plan": protocol.PLAN_PATH, "heartbeat": protocol.HEARTBEAT_PATH, "result": protocol.RESULT_PATH, diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index bc770e5c..d7893bd5 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -2,6 +2,7 @@ import asyncio import json +import threading import time from dataclasses import replace from pathlib import Path @@ -9,11 +10,14 @@ import pytest from failproofai_sdk.evaluator import ( + AssignmentDefinition, ClaimResponse, ConditionResult, + DefinitionsResponse, EvalResult, Evaluator, EvaluatorAPIError, + ExecutionMode, HeartbeatResponse, PlannedRun, PlanResponse, @@ -23,6 +27,7 @@ SessionTranscript, WorkerConfig, WorkerRuntime, + source_checksum, ) FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" @@ -38,6 +43,7 @@ def __init__(self): self.assignment = ClaimResponse.from_wire( samples["claim_response"] ).assignments[0] + self.assignment = replace(self.assignment, definitions_url="") self.session = SessionTranscript.from_wire(samples["transcript_response"]) self.register_requests = [] self.claim_requests = [] @@ -94,6 +100,131 @@ def _runtime(evaluator, client): ) +def test_managed_definition_is_fetched_verified_and_executed(): + source = "EvalResult(score=Score(0.75, passed=True), summary='hosted')" + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, source), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=source, + source_checksum=source_checksum(None, source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "succeeded" + assert result.summary == "hosted" + assert result.results[0].numeric_value == 0.75 + + +def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): + evaluator = Evaluator(name="parallel", version="1") + lock = threading.Lock() + active = 0 + peak = 0 + + def measured(_session): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.04) + with lock: + active -= 1 + return EvalResult(score=Score(1)) + + for index in range(5): + evaluator.eval( + f"eval_{index}", + version="1", + when=lambda session, index=index: ( + index < 3 if session.session_id == "session-a" else index >= 3 + ), + )(measured) + + class ParallelClient(FakeClient): + def transcript(self, assignment, *, worker_id): + assert worker_id == "worker-test" + return replace( + self.session, + assignment_id=assignment.assignment_id, + session_id=assignment.session_id, + session_revision_id=assignment.session_revision_id, + ) + + client = ParallelClient() + first = replace( + client.assignment, + assignment_id="assignment-a", + session_id="session-a", + session_revision_id="revision-a", + ) + second = replace( + client.assignment, + assignment_id="assignment-b", + session_id="session-b", + session_revision_id="revision-b", + ) + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + await asyncio.gather( + runtime.process_assignment(first), runtime.process_assignment(second) + ) + + asyncio.run(exercise()) + + assert peak == 2 + assert len(client.submissions) == 5 + assert client.heartbeats + + def test_condition_failures_are_isolated_and_plan_is_declared_first(): evaluator = Evaluator(name="test", version="1") @@ -712,3 +843,5 @@ async def measured(session): ) asyncio.run(runtime.process_assignment(client.assignment)) assert peak == 1 + DefinitionsResponse, + ExecutionMode, diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py new file mode 100644 index 00000000..91e5a7f2 --- /dev/null +++ b/sdk/python/tests/test_evaluator_source.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import pytest + +from failproofai_sdk.evaluator import EvalResult, Score +from failproofai_sdk.evaluator.source import ( + MAX_EVALUATOR_SOURCE_BYTES, + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + + +class Session: + event_count = 3 + + +def test_restricted_expressions_can_evaluate_conditions_and_results(): + assert compile_condition("session.event_count > 0")(Session()) is True + result = compile_evaluator("EvalResult(score=Score(0.75, passed=True))")( + Session() + ) + assert isinstance(result, EvalResult) + assert result.score == Score(0.75, passed=True) + + +@pytest.mark.parametrize( + "source", + [ + "__import__('os').system('id')", + "session.__class__", + "(lambda: 1)()", + "[x for x in ().__class__.__base__.__subclasses__()]", + ], +) +def test_restricted_expressions_reject_escape_primitives(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +def test_restricted_expressions_reject_statements_and_oversized_source(): + with pytest.raises(UnsafeEvaluatorSource, match="one expression"): + compile_evaluator("import os") + with pytest.raises(UnsafeEvaluatorSource, match="exceeds"): + compile_evaluator("x" * (MAX_EVALUATOR_SOURCE_BYTES + 1)) + + +def test_result_and_condition_types_are_checked_at_runtime(): + with pytest.raises(TypeError, match="EvalResult"): + compile_evaluator("True")(Session()) + with pytest.raises(TypeError, match="bool or ConditionResult"): + compile_condition("1")(Session()) + + +def test_source_checksum_covers_condition_and_evaluator_together(): + base = source_checksum(None, "EvalResult()") + assert base == source_checksum(None, "EvalResult()") + assert base != source_checksum("True", "EvalResult()") + assert base != source_checksum(None, "EvalResult(summary='changed')")