diff --git a/apps/dev-playground/evals.config.ts b/apps/dev-playground/evals.config.ts new file mode 100644 index 000000000..6ad057c1b --- /dev/null +++ b/apps/dev-playground/evals.config.ts @@ -0,0 +1,15 @@ +import { defineEvalConfig } from "@databricks/appkit/beta"; + +/** + * Root eval config for the dev-playground. `webServer` lets `appkit agent eval` + * boot the app on demand (reusing an already-running dev server) instead of + * requiring it to be started by hand. + */ +export default defineEvalConfig({ + baseUrl: "http://localhost:8000", + webServer: { + // Monorepo fixture command; a template project would use `npm run dev`. + command: "pnpm --filter=dev-playground dev", + timeoutMs: 90_000, + }, +}); diff --git a/docs/docs/api/appkit/Function.defineEvalConfig.md b/docs/docs/api/appkit/Function.defineEvalConfig.md new file mode 100644 index 000000000..72de6c6e0 --- /dev/null +++ b/docs/docs/api/appkit/Function.defineEvalConfig.md @@ -0,0 +1,17 @@ +# Function: defineEvalConfig() + +```ts +function defineEvalConfig(config: EvalConfig): EvalConfig; +``` + +Define per-directory eval config. Default-export from `evals.config.ts`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `config` | `EvalConfig` | + +## Returns + +`EvalConfig` diff --git a/docs/docs/api/appkit/Function.discoverEvalConfigs.md b/docs/docs/api/appkit/Function.discoverEvalConfigs.md new file mode 100644 index 000000000..14fc5500a --- /dev/null +++ b/docs/docs/api/appkit/Function.discoverEvalConfigs.md @@ -0,0 +1,20 @@ +# Function: discoverEvalConfigs() + +```ts +function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[]; +``` + +Discover the per-agent `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at +`/server/agents//evals/evals.config.ts`. Config is per-agent: +each agent's config applies only to that agent's evals. Agents without a +config file are omitted. Returns a stable, sorted list. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `rootDir` | `string` | + +## Returns + +[`DiscoveredEvalConfig`](Interface.DiscoveredEvalConfig.md)[] diff --git a/docs/docs/api/appkit/Function.findRootEvalConfig.md b/docs/docs/api/appkit/Function.findRootEvalConfig.md new file mode 100644 index 000000000..2b6a4cb1b --- /dev/null +++ b/docs/docs/api/appkit/Function.findRootEvalConfig.md @@ -0,0 +1,20 @@ +# Function: findRootEvalConfig() + +```ts +function findRootEvalConfig(rootDir: string): string | undefined; +``` + +Path to the root `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at +`/evals.config.ts`, or `undefined` when absent. The root config +holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the +per-agent configs found by [discoverEvalConfigs](Function.discoverEvalConfigs.md). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `rootDir` | `string` | + +## Returns + +`string` \| `undefined` diff --git a/docs/docs/api/appkit/Function.formatResultsJUnit.md b/docs/docs/api/appkit/Function.formatResultsJUnit.md new file mode 100644 index 000000000..de0e3df36 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatResultsJUnit.md @@ -0,0 +1,20 @@ +# Function: formatResultsJUnit() + +```ts +function formatResultsJUnit(results: EvalResult[]): string; +``` + +Render results as JUnit XML for standard CI test reporters: a single +`` with one `` per result. +Failures carry a `` (error or failing-gate summary); skips a +``. All attribute/text values are XML-escaped. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatResultsJson.md b/docs/docs/api/appkit/Function.formatResultsJson.md new file mode 100644 index 000000000..7cf11944a --- /dev/null +++ b/docs/docs/api/appkit/Function.formatResultsJson.md @@ -0,0 +1,19 @@ +# Function: formatResultsJson() + +```ts +function formatResultsJson(results: EvalResult[]): string; +``` + +Render results as a machine-readable JSON report (2-space indented): +`{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types — +every field present on a result round-trips. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.loadRootEvalConfig.md b/docs/docs/api/appkit/Function.loadRootEvalConfig.md new file mode 100644 index 000000000..8c505b1be --- /dev/null +++ b/docs/docs/api/appkit/Function.loadRootEvalConfig.md @@ -0,0 +1,20 @@ +# Function: loadRootEvalConfig() + +```ts +function loadRootEvalConfig(rootDir: string): Promise; +``` + +Load the root `evals.config.ts` under `rootDir` (the project root), or return +`undefined` when there is none. This is the run-wide config carrying +`baseUrl`/`webServer`; the CLI reads it to resolve options and manage the +app-under-test lifecycle before calling [runEvalsInDir](Function.runEvalsInDir.md). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `rootDir` | `string` | + +## Returns + +`Promise`\<`EvalConfig` \| `undefined`\> diff --git a/docs/docs/api/appkit/Function.readEvalDataset.md b/docs/docs/api/appkit/Function.readEvalDataset.md new file mode 100644 index 000000000..faf01d49f --- /dev/null +++ b/docs/docs/api/appkit/Function.readEvalDataset.md @@ -0,0 +1,26 @@ +# Function: readEvalDataset() + +```ts +function readEvalDataset(client: WorkspaceClient, options: ReadEvalDatasetOptions): Promise; +``` + +Read a Databricks managed evaluation dataset (a Unity Catalog table with +`inputs`/`expectations` columns) into rows, over the public SQL Statement +Execution API. Reuses SQLWarehouseConnector for submit/poll/transform +— its result transform already JSON-parses string columns into objects, so +`inputs`/`expectations` come back as records whether the table stores them as +JSON strings or structs. + +The Python `mlflow.genai.datasets` API needs a Spark session (no TS +equivalent), so we read the backing table directly. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `client` | [`WorkspaceClient`](Interface.WorkspaceClient.md) | +| `options` | [`ReadEvalDatasetOptions`](Interface.ReadEvalDatasetOptions.md) | + +## Returns + +`Promise`\<[`DatasetRow`](Interface.DatasetRow.md)[]\> diff --git a/docs/docs/api/appkit/Function.resolveWorkspaceClient.md b/docs/docs/api/appkit/Function.resolveWorkspaceClient.md new file mode 100644 index 000000000..49d9fe00b --- /dev/null +++ b/docs/docs/api/appkit/Function.resolveWorkspaceClient.md @@ -0,0 +1,21 @@ +# Function: resolveWorkspaceClient() + +```ts +function resolveWorkspaceClient(options: ResolveDatabricksAuthOptions): WorkspaceClient | undefined; +``` + +Construct a Databricks `WorkspaceClient` for the eval runner — the object the +SDK-backed connectors (e.g. `SQLWarehouseConnector`) take. An explicit +host+token builds a PAT client; otherwise the profile (or ambient config) is +used and the SDK resolves credentials, minting OAuth as needed. Returns +`undefined` if construction throws (missing/invalid config). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ResolveDatabricksAuthOptions`](Interface.ResolveDatabricksAuthOptions.md) | + +## Returns + +[`WorkspaceClient`](Interface.WorkspaceClient.md) \| `undefined` diff --git a/docs/docs/api/appkit/Function.runWithRetries.md b/docs/docs/api/appkit/Function.runWithRetries.md new file mode 100644 index 000000000..5b4660a74 --- /dev/null +++ b/docs/docs/api/appkit/Function.runWithRetries.md @@ -0,0 +1,22 @@ +# Function: runWithRetries() + +```ts +function runWithRetries(retries: number, attempt: (attemptNumber: number) => Promise): Promise; +``` + +Run `attempt` up to `1 + retries` times, stopping as soon as it returns a +result without an `error` (infra failures — thrown errors or timeouts — set +`error`; assertion failures do not, so a failed-but-completed eval is returned +on the first try and never retried). Returns the last result when every +attempt errored. `retries` below 0 is treated as 0. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `retries` | `number` | +| `attempt` | (`attemptNumber`: `number`) => `Promise`\<[`EvalResult`](Interface.EvalResult.md)\> | + +## Returns + +`Promise`\<[`EvalResult`](Interface.EvalResult.md)\> diff --git a/docs/docs/api/appkit/Function.userTurns.md b/docs/docs/api/appkit/Function.userTurns.md new file mode 100644 index 000000000..1ad9c8706 --- /dev/null +++ b/docs/docs/api/appkit/Function.userTurns.md @@ -0,0 +1,26 @@ +# Function: userTurns() + +```ts +function userTurns(input: Record): string[]; +``` + +Extract every user-message content, in order, from an MLflow +`{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can +carry a full multi-turn conversation; replaying these against one thread (one +`t.send` per returned string) lets the agent see the accumulating history. + +Only `role === "user"` turns are returned — any interleaved `assistant`/ +`system` messages in the row are ignored, since the agent generates its own +responses; you never inject the dataset's assistant turns. A single-user-turn +row yields a one-element array (backward compatible); a row with no `messages` +yields `[]`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `input` | `Record`\<`string`, `unknown`\> | + +## Returns + +`string`[] diff --git a/docs/docs/api/appkit/Interface.AgentsPluginConfig.md b/docs/docs/api/appkit/Interface.AgentsPluginConfig.md index a2f2c963f..2f888b9d0 100644 --- a/docs/docs/api/appkit/Interface.AgentsPluginConfig.md +++ b/docs/docs/api/appkit/Interface.AgentsPluginConfig.md @@ -260,6 +260,23 @@ are discovered at boot and on `reload()` and read as the service principal. *** +### streamConfig? + +```ts +optional streamConfig: StreamConfig; +``` + +SSE stream configuration for this plugin's `executeStream()` calls (buffer +sizes, `maxEventSize`, TTL, heartbeat). Sets the plugin's StreamManager +defaults; a per-call `stream` config still overrides these. Use it to raise +`maxEventSize` above the 5 MiB default when a stream emits larger events. + +#### Inherited from + +[`BasePluginConfig`](Interface.BasePluginConfig.md).[`streamConfig`](Interface.BasePluginConfig.md#streamconfig) + +*** + ### telemetry? ```ts diff --git a/docs/docs/api/appkit/Interface.AssertionHandle.md b/docs/docs/api/appkit/Interface.AssertionHandle.md index 02e3c746b..0e640960b 100644 --- a/docs/docs/api/appkit/Interface.AssertionHandle.md +++ b/docs/docs/api/appkit/Interface.AssertionHandle.md @@ -12,7 +12,9 @@ metric; `.atLeast(n)` is a soft, score-thresholded assertion. atLeast(threshold: number): AssertionHandle; ``` -Soft assertion that passes only when the score is at least `threshold`. +Set the pass threshold for a scored assertion: it passes only when the +score is at least `threshold`. Keeps the current severity (gate unless also +chained with `.soft()`). #### Parameters diff --git a/docs/docs/api/appkit/Interface.BasePluginConfig.md b/docs/docs/api/appkit/Interface.BasePluginConfig.md index a109fd560..9ee98474b 100644 --- a/docs/docs/api/appkit/Interface.BasePluginConfig.md +++ b/docs/docs/api/appkit/Interface.BasePluginConfig.md @@ -32,6 +32,19 @@ optional name: string; *** +### streamConfig? + +```ts +optional streamConfig: StreamConfig; +``` + +SSE stream configuration for this plugin's `executeStream()` calls (buffer +sizes, `maxEventSize`, TTL, heartbeat). Sets the plugin's StreamManager +defaults; a per-call `stream` config still overrides these. Use it to raise +`maxEventSize` above the 5 MiB default when a stream emits larger events. + +*** + ### telemetry? ```ts diff --git a/docs/docs/api/appkit/Interface.DatasetRow.md b/docs/docs/api/appkit/Interface.DatasetRow.md new file mode 100644 index 000000000..32644c2dc --- /dev/null +++ b/docs/docs/api/appkit/Interface.DatasetRow.md @@ -0,0 +1,22 @@ +# Interface: DatasetRow + +One row of a managed evaluation dataset. `inputs` are the kwargs passed to the +agent for the turn; `expectations` (when present) is the row's ground truth / +guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai` +datasets and of the Unity Catalog table backing a managed eval dataset. + +## Properties + +### expectations? + +```ts +optional expectations: Record; +``` + +*** + +### inputs + +```ts +inputs: Record; +``` diff --git a/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md b/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md new file mode 100644 index 000000000..0f75d6015 --- /dev/null +++ b/docs/docs/api/appkit/Interface.DiscoveredEvalConfig.md @@ -0,0 +1,23 @@ +# Interface: DiscoveredEvalConfig + +A per-agent `evals.config.ts` found under `server/agents//evals/`. + +## Properties + +### agent + +```ts +agent: string; +``` + +The agent id whose evals this config applies to. + +*** + +### file + +```ts +file: string; +``` + +Absolute path to the `evals.config.ts` file. diff --git a/docs/docs/api/appkit/Interface.DriveResult.md b/docs/docs/api/appkit/Interface.DriveResult.md index 15625c1ac..f168e875a 100644 --- a/docs/docs/api/appkit/Interface.DriveResult.md +++ b/docs/docs/api/appkit/Interface.DriveResult.md @@ -34,6 +34,31 @@ Whether the turn completed without an agent/stream error. *** +### toolCallDetails + +```ts +toolCallDetails: { + args: Record; + name: string; +}[]; +``` + +Tool calls with their parsed arguments, in call order. + +#### args + +```ts +args: Record; +``` + +#### name + +```ts +name: string; +``` + +*** + ### toolCalls ```ts diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md index 3bf035507..cdaa2ea5e 100644 --- a/docs/docs/api/appkit/Interface.EvalDefinition.md +++ b/docs/docs/api/appkit/Interface.EvalDefinition.md @@ -14,6 +14,34 @@ Target agent id. Defaults to the eval's parent `server/agents/` dir. *** +### dataset? + +```ts +optional dataset: { + limit?: number; + table: string; +}; +``` + +Run this eval once per row of a Databricks managed evaluation dataset (a +Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns). +Each row is bound to `t.input`/`t.expected`. Requires the runner to have a +workspace client + warehouse (`--warehouse`). Omit for a single-run eval. + +#### limit? + +```ts +optional limit: number; +``` + +#### table + +```ts +table: string; +``` + +*** + ### description? ```ts @@ -22,6 +50,27 @@ optional description: string; Short human description, shown in reports. +*** + +### tags? + +```ts +optional tags: string[]; +``` + +Free-form tags for filtering (see the runner's `tags` / `--tag` option). + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Per-eval timeout (ms): `runEval` races the test against it and records a +non-passing result instead of hanging. Overrides the runner/CLI default. + ## Methods ### test() diff --git a/docs/docs/api/appkit/Interface.EvalDriver.md b/docs/docs/api/appkit/Interface.EvalDriver.md index 69d81a05a..6cf961dda 100644 --- a/docs/docs/api/appkit/Interface.EvalDriver.md +++ b/docs/docs/api/appkit/Interface.EvalDriver.md @@ -5,6 +5,21 @@ app's agents endpoint; future drivers (in-process) implement the same shape. ## Methods +### reset()? + +```ts +optional reset(): void; +``` + +Drop the current conversation so the next `send` starts a fresh thread. +Optional: drivers without a session concept omit it. + +#### Returns + +`void` + +*** + ### send() ```ts diff --git a/docs/docs/api/appkit/Interface.EvalSummary.md b/docs/docs/api/appkit/Interface.EvalSummary.md index 3c8fc1e6e..11a682148 100644 --- a/docs/docs/api/appkit/Interface.EvalSummary.md +++ b/docs/docs/api/appkit/Interface.EvalSummary.md @@ -28,6 +28,16 @@ passed: number; *** +### passRate + +```ts +passRate: number; +``` + +Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). + +*** + ### skipped ```ts diff --git a/docs/docs/api/appkit/Interface.EvalWebServer.md b/docs/docs/api/appkit/Interface.EvalWebServer.md new file mode 100644 index 000000000..f220ad6ae --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalWebServer.md @@ -0,0 +1,49 @@ +# Interface: EvalWebServer + +Auto-start config for the app under test, à la Playwright's `webServer`. When +set in a root `evals.config.ts`, the CLI boots the app before running evals +and tears it down after — so you don't have to start the server by hand. + +## Properties + +### command + +```ts +command: string; +``` + +Shell command that starts the app, e.g. `"npm run dev"`. + +*** + +### reuseExisting? + +```ts +optional reuseExisting: boolean; +``` + +When `true` (default), reuse a server already answering at `url` instead of +spawning one — so a running `dev` server is used as-is. Set `false` to +always spawn a fresh server. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +How long to wait for `url` to answer before giving up. Defaults to 60s. + +*** + +### url? + +```ts +optional url: string; +``` + +URL polled until it answers before evals start. Defaults to the run's +`baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the +server is up). diff --git a/docs/docs/api/appkit/Interface.IAiSearchConfig.md b/docs/docs/api/appkit/Interface.IAiSearchConfig.md index 316b4aac3..679d8b18f 100644 --- a/docs/docs/api/appkit/Interface.IAiSearchConfig.md +++ b/docs/docs/api/appkit/Interface.IAiSearchConfig.md @@ -46,6 +46,23 @@ optional name: string; *** +### streamConfig? + +```ts +optional streamConfig: StreamConfig; +``` + +SSE stream configuration for this plugin's `executeStream()` calls (buffer +sizes, `maxEventSize`, TTL, heartbeat). Sets the plugin's StreamManager +defaults; a per-call `stream` config still overrides these. Use it to raise +`maxEventSize` above the 5 MiB default when a stream emits larger events. + +#### Inherited from + +[`BasePluginConfig`](Interface.BasePluginConfig.md).[`streamConfig`](Interface.BasePluginConfig.md#streamconfig) + +*** + ### telemetry? ```ts diff --git a/docs/docs/api/appkit/Interface.IJobsConfig.md b/docs/docs/api/appkit/Interface.IJobsConfig.md index aeff8fa92..d86e7280c 100644 --- a/docs/docs/api/appkit/Interface.IJobsConfig.md +++ b/docs/docs/api/appkit/Interface.IJobsConfig.md @@ -58,6 +58,23 @@ Poll interval for waitForRun in milliseconds. Defaults to 5000. *** +### streamConfig? + +```ts +optional streamConfig: StreamConfig; +``` + +SSE stream configuration for this plugin's `executeStream()` calls (buffer +sizes, `maxEventSize`, TTL, heartbeat). Sets the plugin's StreamManager +defaults; a per-call `stream` config still overrides these. Use it to raise +`maxEventSize` above the 5 MiB default when a stream emits larger events. + +#### Inherited from + +[`BasePluginConfig`](Interface.BasePluginConfig.md).[`streamConfig`](Interface.BasePluginConfig.md#streamconfig) + +*** + ### telemetry? ```ts diff --git a/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md b/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md new file mode 100644 index 000000000..f6411df51 --- /dev/null +++ b/docs/docs/api/appkit/Interface.ReadEvalDatasetOptions.md @@ -0,0 +1,31 @@ +# Interface: ReadEvalDatasetOptions + +## Properties + +### limit? + +```ts +optional limit: number; +``` + +Optional row cap. + +*** + +### table + +```ts +table: string; +``` + +Fully-qualified UC table: `catalog.schema.table`. + +*** + +### warehouseId + +```ts +warehouseId: string; +``` + +SQL warehouse id to run the read against. diff --git a/docs/docs/api/appkit/Interface.RunEvalOptions.md b/docs/docs/api/appkit/Interface.RunEvalOptions.md index 16a6b8190..f89837405 100644 --- a/docs/docs/api/appkit/Interface.RunEvalOptions.md +++ b/docs/docs/api/appkit/Interface.RunEvalOptions.md @@ -22,6 +22,16 @@ Stable id for the eval (e.g. its file path relative to the evals dir). *** +### row? + +```ts +optional row: DatasetRow; +``` + +Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. + +*** + ### strict? ```ts @@ -29,3 +39,14 @@ optional strict: boolean; ``` When true, soft assertion failures also fail the eval. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this; +when both are unset the eval runs unbounded (current behavior). diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md index 48df08c07..38db8f471 100644 --- a/docs/docs/api/appkit/Interface.RunEvalsOptions.md +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -151,6 +151,19 @@ Progress callback, invoked as evals are discovered, started, and finished. *** +### retries? + +```ts +optional retries: number; +``` + +Re-run an eval up to this many extra times when it fails on an +infrastructure error (a thrown error or timeout — `result.error` set), to +absorb transient turn/stream flakiness. Assertion failures are NEVER +retried (a wrong reply is real signal, not flake). Defaults to `0`. + +*** + ### rootDir? ```ts @@ -171,10 +184,44 @@ Soft assertion failures also fail the eval. *** +### tags? + +```ts +optional tags: string[]; +``` + +Only run evals whose `tags` intersect this list. Empty/undefined runs all. +Tags live on the eval def, so filtering happens after each file is loaded. + +*** + ### timeoutMs? ```ts optional timeoutMs: number; ``` -Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. +Default per-eval timeout (ms): `runEval` races the whole test against it and +it also caps each driver turn. A per-eval `def.timeoutMs` overrides it, and +it wins over an agent's `evals.config.ts` `timeoutMs`. Unbounded when unset. + +*** + +### warehouseId? + +```ts +optional warehouseId: string; +``` + +SQL warehouse id used to read managed evaluation datasets. + +*** + +### workspaceClient? + +```ts +optional workspaceClient: WorkspaceClient; +``` + +Workspace client used to read managed evaluation datasets (for evals that +declare `dataset`). Required alongside [warehouseId](#warehouseid) for those evals. diff --git a/docs/docs/api/appkit/Interface.TestContext.md b/docs/docs/api/appkit/Interface.TestContext.md index e21156235..ab2c549a1 100644 --- a/docs/docs/api/appkit/Interface.TestContext.md +++ b/docs/docs/api/appkit/Interface.TestContext.md @@ -4,6 +4,28 @@ The `t` context passed to an eval's `test` function. ## Properties +### expected + +```ts +readonly expected: Record | undefined; +``` + +The current dataset row's `expectations` (ground truth / guidelines), or +`undefined` when the row has none or the eval isn't dataset-driven. + +*** + +### input + +```ts +readonly input: Record; +``` + +The current dataset row's `inputs` when the eval is dataset-driven (see +[EvalDefinition.dataset](Interface.EvalDefinition.md#dataset)); `{}` for a plain single-run eval. + +*** + ### judge ```ts @@ -15,9 +37,10 @@ judge: { ``` LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge -model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)` -to set the pass threshold or `.gate()` to make it a hard gate. Requires the -judge to be configured (`--judge-model`). +model). Each returns a scored assertion that gates by default (a miss fails +the eval); chain `.atLeast(n)` to change the pass threshold or `.soft()` to +demote to a tracked-only metric. Requires the judge to be configured +(`--judge-model`). #### closedQA() @@ -125,6 +148,29 @@ Assert a tool was called during the run (gate by default). *** +### calledToolWith() + +```ts +calledToolWith(name: string, expected: Record): AssertionHandle; +``` + +Assert a tool was called with arguments that deep-contain `expected`: every +key in `expected` must equal the actual argument (recursively for nested +objects), so extra arguments are ignored. Gate by default. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | +| `expected` | `Record`\<`string`, `unknown`\> | + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) + +*** + ### check() ```ts @@ -146,6 +192,22 @@ Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. *** +### reset() + +```ts +reset(): void; +``` + +Start a fresh conversation: the next `send` opens a new thread with no +history. Use to run several independent one-shot checks in one test. +Consecutive `send`s (without a `reset`) stay in one multi-turn conversation. + +#### Returns + +`void` + +*** + ### send() ```ts diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 42afc4a50..081dfef4c 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -51,7 +51,9 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | | [DatabaseRegistry](Interface.DatabaseRegistry.md) | CANONICAL augmentation target. Empty by default; the generated `database.d.ts` augments it via `declare module "@databricks/appkit" { interface DatabaseRegistry { ... } }`. | | [DatabricksAuth](Interface.DatabricksAuth.md) | Resolved Databricks host + bearer token for the eval runner's REST calls. | +| [DatasetRow](Interface.DatasetRow.md) | One row of a managed evaluation dataset. `inputs` are the kwargs passed to the agent for the turn; `expectations` (when present) is the row's ground truth / guidelines. Mirrors the `{inputs, expectations}` shape of `mlflow.genai` datasets and of the Unity Catalog table backing a managed eval dataset. | | [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `server/agents//evals/`. | +| [DiscoveredEvalConfig](Interface.DiscoveredEvalConfig.md) | A per-agent `evals.config.ts` found under `server/agents//evals/`. | | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | | [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. | @@ -59,6 +61,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [EvalResult](Interface.EvalResult.md) | The outcome of running one eval. | | [EvalRunSummary](Interface.EvalRunSummary.md) | - | | [EvalSummary](Interface.EvalSummary.md) | - | +| [EvalWebServer](Interface.EvalWebServer.md) | Auto-start config for the app under test, à la Playwright's `webServer`. When set in a root `evals.config.ts`, the CLI boots the app before running evals and tears it down after — so you don't have to start the server by hand. | | [FilePolicyUser](Interface.FilePolicyUser.md) | Minimal user identity passed to the policy function. | | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. | | [FunctionTool](Interface.FunctionTool.md) | - | @@ -85,6 +88,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [PluginToolkitProvider](Interface.PluginToolkitProvider.md) | Minimum shape every entry in the [Plugins](TypeAlias.Plugins.md) map must expose. Core plugins (analytics, files, genie, lakebase) implement this directly via their `.toolkit()` method. The agents plugin and standalone `runAgent` synthesize this shape for any registered plugin that doesn't implement `.toolkit()` directly (falling back to `getAgentTools()` walking). | | [PostResult](Interface.PostResult.md) | Structured result for a best-effort POST that must not throw. | | [PromptContext](Interface.PromptContext.md) | Context passed to `baseSystemPrompt` callbacks. | +| [ReadEvalDatasetOptions](Interface.ReadEvalDatasetOptions.md) | - | | [RegisteredAgent](Interface.RegisteredAgent.md) | - | | [ReportOutcome](Interface.ReportOutcome.md) | - | | [RequestedClaims](Interface.RequestedClaims.md) | Optional claims for fine-grained Unity Catalog table permissions When specified, the returned token will be scoped to only the requested tables | @@ -185,20 +189,25 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | | [database](Function.database.md) | Create a typed database plugin registration for a finalized schema. | | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `server/agents//evals/*.eval.ts` file. | +| [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. | | [defineManifest](Function.defineManifest.md) | Validates a raw manifest (typically a `manifest.json` import) against the canonical Zod schema and returns it as a strict [PluginManifest](Interface.PluginManifest.md). | | [defineSchema](Function.defineSchema.md) | Compile one declared schema. The returned type keeps the table names the builder returned, so `crudRoutes` and `hooks` can name only real tables. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | +| [discoverEvalConfigs](Function.discoverEvalConfigs.md) | Discover the per-agent `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at `/server/agents//evals/evals.config.ts`. Config is per-agent: each agent's config applies only to that agent's evals. Agents without a config file are omitted. Returns a stable, sorted list. | | [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/server/agents//evals/` — co-located with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents plugin discovers). The agent id is the folder name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. | | [enumColumn](Function.enumColumn.md) | - | | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | | [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. | | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. | | [extractServingEndpoints](Function.extractServingEndpoints.md) | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings. | +| [findRootEvalConfig](Function.findRootEvalConfig.md) | Path to the root `evals.config.ts` (from [defineEvalConfig](Function.defineEvalConfig.md)) at `/evals.config.ts`, or `undefined` when absent. The root config holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the per-agent configs found by [discoverEvalConfigs](Function.discoverEvalConfigs.md). | | [findServerFile](Function.findServerFile.md) | Find the server entry file by checking candidate paths in order. | | [fk](Function.fk.md) | Declare foreign-key to another column. | | [formatEvalDetail](Function.formatEvalDetail.md) | Indented detail lines for a failing eval (error + failing assertions). | | [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). | | [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). | +| [formatResultsJson](Function.formatResultsJson.md) | Render results as a machine-readable JSON report (2-space indented): `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types — every field present on a result round-trips. | +| [formatResultsJUnit](Function.formatResultsJUnit.md) | Render results as JUnit XML for standard CI test reporters: a single `` with one `` per result. Failures carry a `` (error or failing-gate summary); skips a ``. All attribute/text values are XML-escaped. | | [formatSummaryLine](Function.formatSummaryLine.md) | The final PASS/FAIL summary line. | | [fromSupervisorApi](Function.fromSupervisorApi.md) | Creates an [AgentAdapter](Interface.AgentAdapter.md) backed by the Databricks AI Gateway Responses API (`/ai-gateway/mlflow/v1/responses`). | | [functionToolToDefinition](Function.functionToolToDefinition.md) | - | @@ -222,20 +231,25 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [jsonb](Function.jsonb.md) | - | | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. | | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. | +| [loadRootEvalConfig](Function.loadRootEvalConfig.md) | Load the root `evals.config.ts` under `rootDir` (the project root), or return `undefined` when there is none. This is the run-wide config carrying `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the app-under-test lifecycle before calling [runEvalsInDir](Function.runEvalsInDir.md). | | [matches](Function.matches.md) | Passes when the value matches `pattern`. | | [mcpServer](Function.mcpServer.md) | Factory for declaring a custom MCP server tool. | | [normalizeHost](Function.normalizeHost.md) | Ensure the host has a scheme (Databricks env often lacks `https://`). | | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. | +| [readEvalDataset](Function.readEvalDataset.md) | Read a Databricks managed evaluation dataset (a Unity Catalog table with `inputs`/`expectations` columns) into rows, over the public SQL Statement Execution API. Reuses SQLWarehouseConnector for submit/poll/transform — its result transform already JSON-parses string columns into objects, so `inputs`/`expectations` come back as records whether the table stores them as JSON strings or structs. | | [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. | | [resolveDatabricksAuth](Function.resolveDatabricksAuth.md) | - | | [resolveHostedTools](Function.resolveHostedTools.md) | - | +| [resolveWorkspaceClient](Function.resolveWorkspaceClient.md) | Construct a Databricks `WorkspaceClient` for the eval runner — the object the SDK-backed connectors (e.g. `SQLWarehouseConnector`) take. An explicit host+token builds a PAT client; otherwise the profile (or ambient config) is used and the SDK resolves credentials, minting OAuth as needed. Returns `undefined` if construction throws (missing/invalid config). | | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. | | [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. | | [runEvalsInDir](Function.runEvalsInDir.md) | Discover, load, and run every eval under each agent's `evals/` dir, driving the agents on a running app. Never throws for an individual eval — load/run failures become non-passing [EvalResult](Interface.EvalResult.md)s. | +| [runWithRetries](Function.runWithRetries.md) | Run `attempt` up to `1 + retries` times, stopping as soon as it returns a result without an `error` (infra failures — thrown errors or timeouts — set `error`; assertion failures do not, so a failed-but-completed eval is returned on the first try and never retried). Returns the last result when every attempt errored. `retries` below 0 is treated as 0. | | [summarize](Function.summarize.md) | - | | [text](Function.text.md) | - | | [timestamp](Function.timestamp.md) | - | | [tool](Function.tool.md) | Factory for defining function tools with Zod schemas. | | [toolsFromRegistry](Function.toolsFromRegistry.md) | Produces the `AgentToolDefinition[]` a ToolProvider exposes to the LLM, deriving `parameters` JSON Schema from each entry's Zod schema. | +| [userTurns](Function.userTurns.md) | Extract every user-message content, in order, from an MLflow `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can carry a full multi-turn conversation; replaying these against one thread (one `t.send` per returned string) lets the agent see the accumulating history. | | [uuid](Function.uuid.md) | - | | [varchar](Function.varchar.md) | - | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index 6dd11de11..ba4fbf41b 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -187,11 +187,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.DatabricksAuth", label: "DatabricksAuth" }, + { + type: "doc", + id: "api/appkit/Interface.DatasetRow", + label: "DatasetRow" + }, { type: "doc", id: "api/appkit/Interface.DiscoveredEval", label: "DiscoveredEval" }, + { + type: "doc", + id: "api/appkit/Interface.DiscoveredEvalConfig", + label: "DiscoveredEvalConfig" + }, { type: "doc", id: "api/appkit/Interface.DriveResult", @@ -227,6 +237,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.EvalSummary", label: "EvalSummary" }, + { + type: "doc", + id: "api/appkit/Interface.EvalWebServer", + label: "EvalWebServer" + }, { type: "doc", id: "api/appkit/Interface.FilePolicyUser", @@ -357,6 +372,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.PromptContext", label: "PromptContext" }, + { + type: "doc", + id: "api/appkit/Interface.ReadEvalDatasetOptions", + label: "ReadEvalDatasetOptions" + }, { type: "doc", id: "api/appkit/Interface.RegisteredAgent", @@ -800,6 +820,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.defineEval", label: "defineEval" }, + { + type: "doc", + id: "api/appkit/Function.defineEvalConfig", + label: "defineEvalConfig" + }, { type: "doc", id: "api/appkit/Function.defineManifest", @@ -815,6 +840,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.defineTool", label: "defineTool" }, + { + type: "doc", + id: "api/appkit/Function.discoverEvalConfigs", + label: "discoverEvalConfigs" + }, { type: "doc", id: "api/appkit/Function.discoverEvalFiles", @@ -845,6 +875,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.extractServingEndpoints", label: "extractServingEndpoints" }, + { + type: "doc", + id: "api/appkit/Function.findRootEvalConfig", + label: "findRootEvalConfig" + }, { type: "doc", id: "api/appkit/Function.findServerFile", @@ -870,6 +905,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.formatEvalResults", label: "formatEvalResults" }, + { + type: "doc", + id: "api/appkit/Function.formatResultsJson", + label: "formatResultsJson" + }, + { + type: "doc", + id: "api/appkit/Function.formatResultsJUnit", + label: "formatResultsJUnit" + }, { type: "doc", id: "api/appkit/Function.formatSummaryLine", @@ -985,6 +1030,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.loadAgentsFromDir", label: "loadAgentsFromDir" }, + { + type: "doc", + id: "api/appkit/Function.loadRootEvalConfig", + label: "loadRootEvalConfig" + }, { type: "doc", id: "api/appkit/Function.matches", @@ -1005,6 +1055,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.parseTextToolCalls", label: "parseTextToolCalls" }, + { + type: "doc", + id: "api/appkit/Function.readEvalDataset", + label: "readEvalDataset" + }, { type: "doc", id: "api/appkit/Function.reportToMlflow", @@ -1020,6 +1075,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.resolveHostedTools", label: "resolveHostedTools" }, + { + type: "doc", + id: "api/appkit/Function.resolveWorkspaceClient", + label: "resolveWorkspaceClient" + }, { type: "doc", id: "api/appkit/Function.runAgent", @@ -1035,6 +1095,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.runEvalsInDir", label: "runEvalsInDir" }, + { + type: "doc", + id: "api/appkit/Function.runWithRetries", + label: "runWithRetries" + }, { type: "doc", id: "api/appkit/Function.summarize", @@ -1060,6 +1125,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.toolsFromRegistry", label: "toolsFromRegistry" }, + { + type: "doc", + id: "api/appkit/Function.userTurns", + label: "userTurns" + }, { type: "doc", id: "api/appkit/Function.uuid", diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts index b6c8b6e88..dd0d1cec1 100644 --- a/packages/appkit/src/evals/discover.ts +++ b/packages/appkit/src/evals/discover.ts @@ -76,6 +76,17 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { ); } +/** + * Path to the root `evals.config.ts` (from {@link defineEvalConfig}) at + * `/evals.config.ts`, or `undefined` when absent. The root config + * holds run-wide settings (`baseUrl`, `webServer`); it's distinct from the + * per-agent configs found by {@link discoverEvalConfigs}. + */ +export function findRootEvalConfig(rootDir: string): string | undefined { + const file = path.join(rootDir, "evals.config.ts"); + return existsSync(file) ? file : undefined; +} + /** * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at * `/server/agents//evals/evals.config.ts`. Config is per-agent: diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index afed3046a..9c0e539bf 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -19,6 +19,7 @@ export { type DiscoveredEvalConfig, discoverEvalConfigs, discoverEvalFiles, + findRootEvalConfig, } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; export { @@ -49,6 +50,7 @@ export { type RunEvalOptions, runEval } from "./run-eval"; export { type EvalProgress, type EvalRunSummary, + loadRootEvalConfig, type RunEvalsOptions, runEvalsInDir, runWithRetries, @@ -61,6 +63,7 @@ export type { EvalDefinition, EvalDriver, EvalResult, + EvalWebServer, Matcher, MatchResult, Severity, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 2ad9e4bcd..68c9b3cf7 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -7,6 +7,7 @@ import { type DiscoveredEval, discoverEvalConfigs, discoverEvalFiles, + findRootEvalConfig, } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge, teardownJudge } from "./judge"; @@ -136,6 +137,20 @@ async function loadEvalConfig(file: string): Promise { return resolveConfigDefault(mod); } +/** + * Load the root `evals.config.ts` under `rootDir` (the project root), or return + * `undefined` when there is none. This is the run-wide config carrying + * `baseUrl`/`webServer`; the CLI reads it to resolve options and manage the + * app-under-test lifecycle before calling {@link runEvalsInDir}. + */ +export async function loadRootEvalConfig( + rootDir: string, +): Promise { + const file = findRootEvalConfig(rootDir); + if (!file) return undefined; + return loadEvalConfig(file); +} + /** * Unwrap the config default export across module-interop shapes (see * {@link resolveEvalDefault}). A config has no `.test`, so the first plain diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts index cd2d949af..4d51762a0 100644 --- a/packages/appkit/src/evals/tests/discover.test.ts +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -4,7 +4,11 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; -import { discoverEvalConfigs, discoverEvalFiles } from "../discover"; +import { + discoverEvalConfigs, + discoverEvalFiles, + findRootEvalConfig, +} from "../discover"; let root: string; @@ -61,3 +65,15 @@ describe("discoverEvalConfigs", () => { expect(discoverEvalConfigs(root)).toEqual([]); }); }); + +describe("findRootEvalConfig", () => { + test("finds a root evals.config.ts", () => { + write("evals.config.ts"); + expect(findRootEvalConfig(root)).toBe(path.join(root, "evals.config.ts")); + }); + + test("returns undefined when absent (and ignores per-agent configs)", () => { + write("server/agents/support/evals/evals.config.ts"); + expect(findRootEvalConfig(root)).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index 463c92533..bd274f5be 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -170,7 +170,39 @@ export interface EvalDefinition { test(t: TestContext): Promise | void; } -/** Per-directory config from `evals.config.ts` (see {@link defineEvalConfig}). */ +/** + * Auto-start config for the app under test, à la Playwright's `webServer`. When + * set in a root `evals.config.ts`, the CLI boots the app before running evals + * and tears it down after — so you don't have to start the server by hand. + */ +export interface EvalWebServer { + /** Shell command that starts the app, e.g. `"npm run dev"`. */ + command: string; + /** + * URL polled until it answers before evals start. Defaults to the run's + * `baseUrl` (`--url`). Readiness = any HTTP response (a 404 still proves the + * server is up). + */ + url?: string; + /** How long to wait for `url` to answer before giving up. Defaults to 60s. */ + timeoutMs?: number; + /** + * When `true` (default), reuse a server already answering at `url` instead of + * spawning one — so a running `dev` server is used as-is. Set `false` to + * always spawn a fresh server. + */ + reuseExisting?: boolean; +} + +/** + * Eval config from `evals.config.ts` (via {@link defineEvalConfig}). + * + * Two scopes share this shape: a **root** `evals.config.ts` (project root) may + * set run-wide settings — `baseUrl` and `webServer` — plus defaults for + * `maxConcurrency`/`timeoutMs`; a **per-agent** `server/agents//evals/evals.config.ts` + * sets only that agent's `maxConcurrency`/`timeoutMs` overrides (`baseUrl`/ + * `webServer` there are ignored — server lifecycle is run-wide). + */ export interface EvalConfig { /** LLM judge config. Defaults to the agent's own serving endpoint. */ judge?: { model?: string }; @@ -178,6 +210,10 @@ export interface EvalConfig { maxConcurrency?: number; /** Default per-eval timeout. */ timeoutMs?: number; + /** Base URL of the app to drive (root config only). Overridden by `--url`. */ + baseUrl?: string; + /** Auto-start the app under test (root config only). */ + webServer?: EvalWebServer; } /** The outcome of running one eval. */ diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index 5bda63492..3bf1d30a9 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -1,3 +1,4 @@ +import { type ChildProcess, spawn } from "node:child_process"; import fs from "node:fs"; import { Command, Option } from "commander"; @@ -55,6 +56,7 @@ interface EvalRunner { token?: string; }): unknown; formatEvalHeadline(result: unknown): string; + loadRootEvalConfig(rootDir: string): Promise; evalGlyph(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; @@ -63,6 +65,19 @@ interface EvalRunner { summarize(results: unknown[]): { allPassed: boolean; passRate: number }; } +/** Subset of `@databricks/appkit/beta`'s `EvalConfig` the CLI reads. */ +interface EvalConfig { + maxConcurrency?: number; + timeoutMs?: number; + baseUrl?: string; + webServer?: { + command: string; + url?: string; + timeoutMs?: number; + reuseExisting?: boolean; + }; +} + /** * Loaded at runtime from the consuming project so this command (which ships in * `@databricks/shared`) doesn't take a build-time dependency on appkit. The @@ -97,8 +112,83 @@ function positiveInt(raw: string | undefined): number | undefined { return n > 0 ? n : undefined; } +/** True when `url` answers with any HTTP response (a 404 still proves it's up). */ +async function isServerUp(url: string): Promise { + try { + await fetch(url, { signal: AbortSignal.timeout(2000) }); + return true; + } catch { + return false; + } +} + +/** + * Start the app under test per a root config's `webServer`, à la Playwright: + * reuse a server already answering at `url` (unless `reuseExisting: false`), + * else spawn `command`, poll `url` until it answers or `timeoutMs` elapses. + * Returns a `stop()` that kills the spawned process group (a no-op when the + * server was reused). Logs go to stderr so a machine reporter's stdout stays + * clean. Throws if the server never comes up. + */ +async function startWebServer( + webServer: NonNullable, + baseUrl: string, +): Promise<{ stop: () => void }> { + const url = webServer.url ?? baseUrl; + const reuse = webServer.reuseExisting !== false; + const noop = { stop: () => {} }; + + if (reuse && (await isServerUp(url))) { + console.error(`Reusing server already running at ${url}`); + return noop; + } + + console.error(`Starting web server: ${webServer.command}`); + // `detached` + a negative-PID kill lets us tear down the whole process group + // (dev servers spawn child processes). stdout/stderr inherit so the user sees + // build output; the server's stdout is not our report stream. + const child: ChildProcess = spawn(webServer.command, { + shell: true, + detached: true, + stdio: "inherit", + }); + + let exited = false; + child.on("exit", () => { + exited = true; + }); + + const stop = (): void => { + if (exited || child.pid === undefined) return; + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + // Group already gone, or never became a leader — best-effort. + } + }; + + const deadline = Date.now() + (webServer.timeoutMs ?? 60_000); + try { + while (Date.now() < deadline) { + if (exited) throw new Error("web server exited before becoming ready"); + if (await isServerUp(url)) { + console.error(`Web server ready at ${url}`); + return { stop }; + } + await new Promise((r) => setTimeout(r, 500)); + } + } catch (err) { + stop(); + throw err; + } + stop(); + throw new Error( + `web server did not respond at ${url} within ${webServer.timeoutMs ?? 60_000}ms`, + ); +} + interface EvalOptions { - url: string; + url?: string; strict?: boolean; root?: string; header?: string[]; @@ -228,6 +318,15 @@ async function runAgentEval( ): Promise { const runner = await loadRunner(); + // Root `evals.config.ts` (project root) carries run-wide settings — baseUrl, + // webServer, and defaults for concurrency/timeout. A CLI flag always wins. + const rootDir = opts.root ?? process.cwd(); + const config = (await runner.loadRootEvalConfig(rootDir)) ?? {}; + + // Base URL: --url flag > config.baseUrl > built-in default. `--url` has no + // commander default so an unset flag is undefined and lets config win. + const baseUrl = opts.url ?? config.baseUrl ?? "http://localhost:8000"; + // Databricks credentials shared by auth resolution and the workspace client: // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI // profile. @@ -247,8 +346,13 @@ async function runAgentEval( const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID; const workspaceClient = runner.resolveWorkspaceClient(credentials); - // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins. - const timeoutMs = positiveInt(opts.timeout); + // Max concurrency: the `--concurrency` flag (already parsed by its argParser) + // wins over the root config's value; else the runner's built-in default. + const concurrency = opts.concurrency ?? config.maxConcurrency; + + // Runner-level default per-eval timeout (ms). --timeout flag wins over the + // root config; a per-eval `timeoutMs` overrides both (applied in the runner). + const timeoutMs = positiveInt(opts.timeout) ?? config.timeoutMs; // Extra attempts for evals that fail on an infra error (turn/timeout). Junk // or negative input falls back to no retries. @@ -264,23 +368,29 @@ async function runAgentEval( else console.log(msg); }; + // Boot the app under test if the root config declares a webServer (reuses an + // already-running server unless told otherwise); always torn down after. + const server = config.webServer + ? await startWebServer(config.webServer, baseUrl) + : undefined; + let summary: EvalRunSummary; try { summary = await runner.runEvalsInDir({ - rootDir: opts.root, - baseUrl: opts.url, + rootDir, + baseUrl, filter, tags: opts.tag, strict: opts.strict, headers: opts.header ? parseHeaders(opts.header) : undefined, - concurrency: opts.concurrency, + concurrency, mlflow: resolveMlflow(opts, auth), judge: resolveJudge(opts, auth), workspaceClient, warehouseId, timeoutMs, retries, - onEvent: makeProgressReporter(runner, opts.url, machine, info), + onEvent: makeProgressReporter(runner, baseUrl, machine, info), }); } catch (err) { // Setup failures (e.g. a bad --experiment for the MLflow run) reject before @@ -291,6 +401,8 @@ async function runAgentEval( ); process.exitCode = 1; return; + } finally { + server?.stop(); } // The final human summary always shows (stderr for machine reporters so it @@ -348,7 +460,10 @@ export const agentEvalCommand = new Command("eval") "[filter]", "Only run evals whose / contains this substring (or an exact agent id)", ) - .option("--url ", "Base URL of the running app", "http://localhost:3000") + .option( + "--url ", + "Base URL of the app to drive (default: evals.config.ts baseUrl, else http://localhost:8000)", + ) .option("--strict", "Fail on soft-assertion misses too", false) .option( "--concurrency ",