diff --git a/AGENTS.md b/AGENTS.md index 14a8e1d..100f3fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -694,6 +694,21 @@ destination is equivalent in configuration but not identical. Acceptable on a dedicated eval project, and worth knowing before pointing any of this at one that is not. +**Outpost validates a destination's shape, not its reachability.** A +well-formed but entirely fictional SQS queue and key pair is accepted; +`queue_url: "not-a-url"` is rejected with `422 "config.queue_url failed pattern +validation"`. That is what makes `outpost-004` scoreable without a live queue: +the API itself covers the part real infrastructure would add least to, and +standing up a real queue would mean cloud credentials inside the agent sandbox +and an external dependency that can fail a run for reasons no agent caused. +Delivery is already proven against webhook destinations elsewhere. + +**Destination credentials are redacted on read** — `AKIA****************` — so a +scorer can assert that credentials were supplied and never which ones. Same +limitation as Hookdeck source `config.auth`. Presence is still worth checking: +an agent thinking in webhook shapes puts the access key in `config` beside the +queue URL and leaves `credentials` empty. + **Reset is to pristine, not to empty.** A new Hookdeck project ships with default issue triggers. The first acquire snapshots what the project contains, and every reset deletes only what a run added. diff --git a/evals/benchmark-outpost-004-queue-destination/EVAL.ts b/evals/benchmark-outpost-004-queue-destination/EVAL.ts new file mode 100644 index 0000000..2146e92 --- /dev/null +++ b/evals/benchmark-outpost-004-queue-destination/EVAL.ts @@ -0,0 +1,250 @@ +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@hookdeck-evals/core'; + +/** + * Can an agent configure a destination that is not a webhook? + * + * Nearly all real Outpost traffic is webhooks, which is exactly why this is + * worth scoring: an agent that has only ever seen the webhook shape reaches for + * a `url` and a secret, and a queue is neither. It is a `type`, a set of config + * fields whose names differ per provider, and `credentials` as a separate + * object. Outpost rejects a malformed `queue_url` with a 422, so the failure is + * loud if they get the field right and the value wrong — and silent if they put + * the value in the wrong field. + * + * **This scores configuration, not delivery, and that is deliberate.** The + * normal rule in this repo is to score behaviour where the API allows it, and + * here it does not: proving delivery would need a real SQS queue, which means + * cloud credentials inside the agent sandbox, an external dependency that can + * fail a run for reasons no agent caused, and queue lifecycle in CI. Delivery + * is already proven against webhook destinations by `outpost-001` and + * `outpost-002`; what is untested is whether an agent can configure a + * non-HTTP type at all. + * + * Measured rather than assumed: Outpost validates the *shape* of a destination + * on create and not its reachability. A well-formed but entirely fictional + * queue and key pair is accepted; `queue_url: "not-a-url"` is rejected with + * `422 "config.queue_url failed pattern validation"`. So the API itself covers + * the part a live queue would add least to. + * + * The trap is in the workspace note rather than the API. Acme want *orders* on + * the queue and everything else unchanged, so deleting the webhook destination + * — the obvious way to "stop sending their orders to the old endpoint" — also + * stops their retry notifications, which nobody asked for. The correct move is + * narrower than the obvious one. + */ + +const TENANT = 'acme'; +const OTHER_TENANT = 'globex'; +/** Exactly what the workspace note gives them. */ +const QUEUE_URL = + 'https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events'; +const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders'; +const ORDERS = 'orders'; +/** The topic they never asked to change, and the one an over-broad fix breaks. */ +const RETRIES = 'retries'; + +interface Destination { + id?: string; + type?: string; + topics?: string[]; + config?: Record; + credentials?: Record; + disabled_at?: string | null; +} + +const scorer: ToolScorer = async (ctx) => { + if (!ctx.outpost) { + throw new Error( + 'no Outpost client, but this scenario declares `requires: [outpost]` ' + + 'and should have been skipped rather than scored' + ); + } + + const destinations = await listDestinations(ctx, TENANT); + const queues = destinations.filter( + (d) => normalise(d.config?.queue_url) === normalise(QUEUE_URL) + ); + const webhooks = destinations.filter( + (d) => normalise(d.config?.url) === normalise(OLD_ENDPOINT) + ); + + const checks: CheckResult[] = [ + checkQueueExists(destinations, queues), + checkQueueReceivesOrders(queues), + checkCredentialsSupplied(queues), + checkOldEndpointStopped(webhooks), + checkRetriesUntouched(webhooks), + await checkOtherTenantUntouched(ctx), + ]; + + return { passed: checks.every((c) => c.passed), checks }; +}; + +export default scorer; + +/** + * Matched on the queue URL rather than on type alone, because "created an SQS + * destination" is not the task — creating the one pointing at *their* queue is. + * An agent that invents a plausible queue has done something worse than + * nothing. + */ +function checkQueueExists( + all: Destination[], + queues: Destination[] +): CheckResult { + const live = queues.filter((d) => !d.disabled_at); + const seen = all + .map( + (d) => `${d.type}:${String(d.config?.queue_url ?? d.config?.url ?? '?')}` + ) + .join(', '); + + if (live.length === 0) { + return { + name: 'their orders are delivered to the queue they gave us', + passed: false, + notes: + queues.length > 0 + ? 'the queue destination exists but is disabled, so nothing reaches it' + : `no enabled destination points at ${QUEUE_URL} (present: ${seen || 'none'})`, + }; + } + + // Type is checked here rather than as its own line: a destination carrying + // their queue URL under a non-queue type is a configuration that cannot work, + // and reporting it as "exists but wrong type" is the useful message. + const sqs = live.filter((d) => d.type === 'aws_sqs'); + return { + name: 'their orders are delivered to the queue they gave us', + passed: sqs.length > 0, + notes: + sqs.length > 0 + ? undefined + : `a destination points at the queue but its type is ` + + `${live.map((d) => d.type).join(', ')} rather than aws_sqs`, + }; +} + +function checkQueueReceivesOrders(queues: Destination[]): CheckResult { + const subscribed = queues.some((d) => subscribes(d, ORDERS)); + return { + name: 'the queue is subscribed to their order events', + passed: subscribed, + notes: subscribed + ? undefined + : `the queue destination is not subscribed to ${ORDERS} ` + + `(topics: ${topicsOf(queues)}), so it would sit empty`, + }; +} + +/** + * Presence only, and it cannot be more than that: credentials are redacted on + * read (`AKIA****************`), so a scorer cannot tell a correct key from a + * plausible one. The same limitation applies to Hookdeck source `config.auth`. + * + * Still worth a line. An agent that puts the access key into `config` alongside + * the queue URL, which is the natural mistake if you are thinking in webhook + * shapes, leaves `credentials` empty and fails here. + */ +function checkCredentialsSupplied(queues: Destination[]): CheckResult { + const withCredentials = queues.filter( + (d) => Object.keys(d.credentials ?? {}).length > 0 + ); + return { + name: 'the queue destination carries credentials', + passed: withCredentials.length > 0, + notes: + withCredentials.length > 0 + ? undefined + : 'no credentials on the queue destination — the access key and secret ' + + 'go in `credentials`, not in `config` beside the queue URL', + }; +} + +function checkOldEndpointStopped(webhooks: Destination[]): CheckResult { + const stillSending = webhooks.filter( + (d) => !d.disabled_at && subscribes(d, ORDERS) + ); + return { + name: 'their orders no longer go to the old endpoint', + passed: stillSending.length === 0, + notes: + stillSending.length === 0 + ? undefined + : 'the old webhook endpoint is still subscribed to orders, so every order ' + + 'is now delivered twice — to the queue and to the endpoint they asked us ' + + 'to stop using', + }; +} + +/** + * The check the scenario turns on. + * + * "Stop sending their orders to the old endpoint" is most simply achieved by + * deleting the webhook destination, and that is wrong: it also stops their + * retry notifications, which the note says should carry on unchanged. Scoring + * only the requested change would pass an agent that broke something adjacent — + * the failure mode `alerting-001` was corrected for. + */ +function checkRetriesUntouched(webhooks: Destination[]): CheckResult { + const stillReceiving = webhooks.filter( + (d) => !d.disabled_at && subscribes(d, RETRIES) + ); + return { + name: 'the rest of their delivery is unchanged', + passed: stillReceiving.length > 0, + notes: + stillReceiving.length > 0 + ? undefined + : `their ${RETRIES} events no longer reach the old endpoint either — moving ` + + 'orders to the queue was not supposed to change anything else', + }; +} + +async function checkOtherTenantUntouched( + ctx: ToolEvalContext +): Promise { + const name = 'the other customer was left alone'; + const destinations = await listDestinations(ctx, OTHER_TENANT); + const live = destinations.filter((d) => !d.disabled_at); + return { + name, + passed: live.length > 0, + notes: + live.length > 0 + ? undefined + : `${OTHER_TENANT} has no working destination left, and they asked for nothing`, + }; +} + +/** `*` subscribes to everything. */ +function subscribes(destination: Destination, topic: string): boolean { + const topics = destination.topics ?? []; + return topics.includes('*') || topics.includes(topic); +} + +function topicsOf(destinations: Destination[]): string { + const topics = destinations.flatMap((d) => d.topics ?? []); + return topics.length > 0 ? topics.join(', ') : 'none'; +} + +function normalise(value: unknown): string { + return typeof value === 'string' ? value.trim().replace(/\/+$/, '') : ''; +} + +/** Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. */ +async function listDestinations( + ctx: ToolEvalContext, + tenantId: string +): Promise { + const rows = await ctx.outpost?.( + 'GET', + `/tenants/${encodeURIComponent(tenantId)}/destinations` + ); + if (!rows) return []; + return Array.isArray(rows) ? rows : (rows.models ?? []); +} diff --git a/evals/benchmark-outpost-004-queue-destination/PROMPT.md b/evals/benchmark-outpost-004-queue-destination/PROMPT.md new file mode 100644 index 0000000..75621a9 --- /dev/null +++ b/evals/benchmark-outpost-004-queue-destination/PROMPT.md @@ -0,0 +1,20 @@ +--- +stage: build +suite: benchmark +gated_by: mixed +product: + - outpost +topic: + - capabilities +requires: + - outpost +motivation: Most Outpost traffic is webhooks, but delivering to a queue is a core capability and a different job — a type rather than a URL, credentials rather than a secret, and fields whose names differ per provider. This scores whether an agent can configure a non-HTTP destination from details it has to find, rather than reaching for the webhook shape it has seen most often. +--- + +Acme are moving off webhooks. Their endpoint keeps falling over under load and +they'd rather we drop order events straight onto a queue they already run. + +They sent us the queue details last week — whoever picked up the ticket put +them in the repo. + +Set that up, and stop sending their orders to the old endpoint. diff --git a/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts b/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts new file mode 100644 index 0000000..1af1aef --- /dev/null +++ b/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts @@ -0,0 +1,60 @@ +import type { ToolEvalContext } from '@hookdeck-evals/core'; + +/** + * What a correct agent leaves behind. + * + * Two changes, and the second is the one worth reading: the webhook destination + * is *narrowed* to the topics acme did not ask to move, rather than deleted. + * Deleting it is the obvious way to stop sending orders there and takes their + * retry notifications with it. + */ + +const TENANT = 'acme'; +const QUEUE_URL = + 'https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events'; +const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders'; + +interface Destination { + id?: string; + config?: Record; + topics?: string[]; +} + +export default async function solve(ctx: ToolEvalContext): Promise { + const outpost = ctx.outpost; + if (!outpost) { + throw new Error( + 'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY' + ); + } + + await outpost('POST', `/tenants/${TENANT}/destinations`, { + type: 'aws_sqs', + topics: ['orders'], + // The queue URL is config; the key pair is credentials. Keeping them + // separate is the whole shape difference between this and a webhook. + config: { queue_url: QUEUE_URL }, + credentials: { + key: 'AKIAIOSFODNN7EXAMPLE', + secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + }, + }); + + const rows = await outpost( + 'GET', + `/tenants/${TENANT}/destinations` + ); + const destinations = Array.isArray(rows) ? rows : (rows.models ?? []); + + for (const destination of destinations) { + if (destination.config?.url !== OLD_ENDPOINT || !destination.id) continue; + const remaining = (destination.topics ?? []).filter((t) => t !== 'orders'); + await outpost( + 'PATCH', + `/tenants/${TENANT}/destinations/${destination.id}`, + { + topics: remaining, + } + ); + } +} diff --git a/evals/benchmark-outpost-004-queue-destination/local/INFRA.md b/evals/benchmark-outpost-004-queue-destination/local/INFRA.md new file mode 100644 index 0000000..7580796 --- /dev/null +++ b/evals/benchmark-outpost-004-queue-destination/local/INFRA.md @@ -0,0 +1,26 @@ +# Customer infrastructure notes + +Details customers have sent us for delivery targets they run themselves. Keep +credentials out of application config — these are here because support needs +them to set delivery up, not because anything reads this file. + +## Acme — order events + +Moving off their webhook endpoint (`https://mock.hookdeck.com/api/v1/acme/orders`), +which has been timing out under load. They want order events on SQS instead. + +Sent over by their platform team on the 14th: + +``` +Queue URL: https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events +Region: eu-west-1 +Access key: AKIAIOSFODNN7EXAMPLE +Secret key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` + +They only want `orders` on the queue. Anything else we send them today should +carry on as it is. + +## Globex — nothing outstanding + +Still on webhooks, happy, no changes requested. diff --git a/evals/benchmark-outpost-004-queue-destination/remote/seed.json b/evals/benchmark-outpost-004-queue-destination/remote/seed.json new file mode 100644 index 0000000..157c295 --- /dev/null +++ b/evals/benchmark-outpost-004-queue-destination/remote/seed.json @@ -0,0 +1,34 @@ +{ + "outpost": { + "tenants": [ + { + "id": "acme", + "topics": ["orders", "retries"], + "destinations": [ + { + "ref": "acme-webhook", + "type": "webhook", + "topics": ["orders", "retries"], + "config": { + "url": "https://mock.hookdeck.com/api/v1/acme/orders" + } + } + ] + }, + { + "id": "globex", + "topics": ["orders"], + "destinations": [ + { + "ref": "globex-webhook", + "type": "webhook", + "topics": ["orders"], + "config": { + "url": "https://mock.hookdeck.com/api/v1/globex/orders" + } + } + ] + } + ] + } +}