Add outpost-004: move a customer from webhooks to a queue

AGENTS.md

@@ -694,6 +694,21 @@ destination is equivalent in configuration but not identical. Acceptable on a
 dedicated eval project, and worth knowing before pointing any of this at one
 that is not.
 
+**Outpost validates a destination's shape, not its reachability.** A
+well-formed but entirely fictional SQS queue and key pair is accepted;
+`queue_url: "not-a-url"` is rejected with `422 "config.queue_url failed pattern
+validation"`. That is what makes `outpost-004` scoreable without a live queue:
+the API itself covers the part real infrastructure would add least to, and
+standing up a real queue would mean cloud credentials inside the agent sandbox
+and an external dependency that can fail a run for reasons no agent caused.
+Delivery is already proven against webhook destinations elsewhere.
+
+**Destination credentials are redacted on read** — `AKIA****************` — so a
+scorer can assert that credentials were supplied and never which ones. Same
+limitation as Hookdeck source `config.auth`. Presence is still worth checking:
+an agent thinking in webhook shapes puts the access key in `config` beside the
+queue URL and leaves `credentials` empty.
+
 **Reset is to pristine, not to empty.** A new Hookdeck project ships with
 default issue triggers. The first acquire snapshots what the project contains,
 and every reset deletes only what a run added.

evals/benchmark-outpost-004-queue-destination/EVAL.ts

@@ -0,0 +1,250 @@
+import type {
+  CheckResult,
+  ToolEvalContext,
+  ToolScorer,
+} from '@hookdeck-evals/core';
+
+/**
+ * Can an agent configure a destination that is not a webhook?
+ *
+ * Nearly all real Outpost traffic is webhooks, which is exactly why this is
+ * worth scoring: an agent that has only ever seen the webhook shape reaches for
+ * a `url` and a secret, and a queue is neither. It is a `type`, a set of config
+ * fields whose names differ per provider, and `credentials` as a separate
+ * object. Outpost rejects a malformed `queue_url` with a 422, so the failure is
+ * loud if they get the field right and the value wrong — and silent if they put
+ * the value in the wrong field.
+ *
+ * **This scores configuration, not delivery, and that is deliberate.** The
+ * normal rule in this repo is to score behaviour where the API allows it, and
+ * here it does not: proving delivery would need a real SQS queue, which means
+ * cloud credentials inside the agent sandbox, an external dependency that can
+ * fail a run for reasons no agent caused, and queue lifecycle in CI. Delivery
+ * is already proven against webhook destinations by `outpost-001` and
+ * `outpost-002`; what is untested is whether an agent can configure a
+ * non-HTTP type at all.
+ *
+ * Measured rather than assumed: Outpost validates the *shape* of a destination
+ * on create and not its reachability. A well-formed but entirely fictional
+ * queue and key pair is accepted; `queue_url: "not-a-url"` is rejected with
+ * `422 "config.queue_url failed pattern validation"`. So the API itself covers
+ * the part a live queue would add least to.
+ *
+ * The trap is in the workspace note rather than the API. Acme want *orders* on
+ * the queue and everything else unchanged, so deleting the webhook destination
+ * — the obvious way to "stop sending their orders to the old endpoint" — also
+ * stops their retry notifications, which nobody asked for. The correct move is
+ * narrower than the obvious one.
+ */
+
+const TENANT = 'acme';
+const OTHER_TENANT = 'globex';
+/** Exactly what the workspace note gives them. */
+const QUEUE_URL =
+  'https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events';
+const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders';
+const ORDERS = 'orders';
+/** The topic they never asked to change, and the one an over-broad fix breaks. */
+const RETRIES = 'retries';
+
+interface Destination {
+  id?: string;
+  type?: string;
+  topics?: string[];
+  config?: Record<string, unknown>;
+  credentials?: Record<string, unknown>;
+  disabled_at?: string | null;
+}
+
+const scorer: ToolScorer = async (ctx) => {
+  if (!ctx.outpost) {
+    throw new Error(
+      'no Outpost client, but this scenario declares `requires: [outpost]` ' +
+        'and should have been skipped rather than scored'
+    );
+  }
+
+  const destinations = await listDestinations(ctx, TENANT);
+  const queues = destinations.filter(
+    (d) => normalise(d.config?.queue_url) === normalise(QUEUE_URL)
+  );
+  const webhooks = destinations.filter(
+    (d) => normalise(d.config?.url) === normalise(OLD_ENDPOINT)
+  );
+
+  const checks: CheckResult[] = [
+    checkQueueExists(destinations, queues),
+    checkQueueReceivesOrders(queues),
+    checkCredentialsSupplied(queues),
+    checkOldEndpointStopped(webhooks),
+    checkRetriesUntouched(webhooks),
+    await checkOtherTenantUntouched(ctx),
+  ];
+
+  return { passed: checks.every((c) => c.passed), checks };
+};
+
+export default scorer;
+
+/**
+ * Matched on the queue URL rather than on type alone, because "created an SQS
+ * destination" is not the task — creating the one pointing at *their* queue is.
+ * An agent that invents a plausible queue has done something worse than
+ * nothing.
+ */
+function checkQueueExists(
+  all: Destination[],
+  queues: Destination[]
+): CheckResult {
+  const live = queues.filter((d) => !d.disabled_at);
+  const seen = all
+    .map(
+      (d) => `${d.type}:${String(d.config?.queue_url ?? d.config?.url ?? '?')}`
+    )
+    .join(', ');
+
+  if (live.length === 0) {
+    return {
+      name: 'their orders are delivered to the queue they gave us',
+      passed: false,
+      notes:
+        queues.length > 0
+          ? 'the queue destination exists but is disabled, so nothing reaches it'
+          : `no enabled destination points at ${QUEUE_URL} (present: ${seen || 'none'})`,
+    };
+  }
+
+  // Type is checked here rather than as its own line: a destination carrying
+  // their queue URL under a non-queue type is a configuration that cannot work,
+  // and reporting it as "exists but wrong type" is the useful message.
+  const sqs = live.filter((d) => d.type === 'aws_sqs');
+  return {
+    name: 'their orders are delivered to the queue they gave us',
+    passed: sqs.length > 0,
+    notes:
+      sqs.length > 0
+        ? undefined
+        : `a destination points at the queue but its type is ` +
+          `${live.map((d) => d.type).join(', ')} rather than aws_sqs`,
+  };
+}
+
+function checkQueueReceivesOrders(queues: Destination[]): CheckResult {
+  const subscribed = queues.some((d) => subscribes(d, ORDERS));
+  return {
+    name: 'the queue is subscribed to their order events',
+    passed: subscribed,
+    notes: subscribed
+      ? undefined
+      : `the queue destination is not subscribed to ${ORDERS} ` +
+        `(topics: ${topicsOf(queues)}), so it would sit empty`,
+  };
+}
+
+/**
+ * Presence only, and it cannot be more than that: credentials are redacted on
+ * read (`AKIA****************`), so a scorer cannot tell a correct key from a
+ * plausible one. The same limitation applies to Hookdeck source `config.auth`.
+ *
+ * Still worth a line. An agent that puts the access key into `config` alongside
+ * the queue URL, which is the natural mistake if you are thinking in webhook
+ * shapes, leaves `credentials` empty and fails here.
+ */
+function checkCredentialsSupplied(queues: Destination[]): CheckResult {
+  const withCredentials = queues.filter(
+    (d) => Object.keys(d.credentials ?? {}).length > 0
+  );
+  return {
+    name: 'the queue destination carries credentials',
+    passed: withCredentials.length > 0,
+    notes:
+      withCredentials.length > 0
+        ? undefined
+        : 'no credentials on the queue destination — the access key and secret ' +
+          'go in `credentials`, not in `config` beside the queue URL',
+  };
+}
+
+function checkOldEndpointStopped(webhooks: Destination[]): CheckResult {
+  const stillSending = webhooks.filter(
+    (d) => !d.disabled_at && subscribes(d, ORDERS)
+  );
+  return {
+    name: 'their orders no longer go to the old endpoint',
+    passed: stillSending.length === 0,
+    notes:
+      stillSending.length === 0
+        ? undefined
+        : 'the old webhook endpoint is still subscribed to orders, so every order ' +
+          'is now delivered twice — to the queue and to the endpoint they asked us ' +
+          'to stop using',
+  };
+}
+
+/**
+ * The check the scenario turns on.
+ *
+ * "Stop sending their orders to the old endpoint" is most simply achieved by
+ * deleting the webhook destination, and that is wrong: it also stops their
+ * retry notifications, which the note says should carry on unchanged. Scoring
+ * only the requested change would pass an agent that broke something adjacent —
+ * the failure mode `alerting-001` was corrected for.
+ */
+function checkRetriesUntouched(webhooks: Destination[]): CheckResult {
+  const stillReceiving = webhooks.filter(
+    (d) => !d.disabled_at && subscribes(d, RETRIES)
+  );
+  return {
+    name: 'the rest of their delivery is unchanged',
+    passed: stillReceiving.length > 0,
+    notes:
+      stillReceiving.length > 0
+        ? undefined
+        : `their ${RETRIES} events no longer reach the old endpoint either — moving ` +
+          'orders to the queue was not supposed to change anything else',
+  };
+}
+
+async function checkOtherTenantUntouched(
+  ctx: ToolEvalContext
+): Promise<CheckResult> {
+  const name = 'the other customer was left alone';
+  const destinations = await listDestinations(ctx, OTHER_TENANT);
+  const live = destinations.filter((d) => !d.disabled_at);
+  return {
+    name,
+    passed: live.length > 0,
+    notes:
+      live.length > 0
+        ? undefined
+        : `${OTHER_TENANT} has no working destination left, and they asked for nothing`,
+  };
+}
+
+/** `*` subscribes to everything. */
+function subscribes(destination: Destination, topic: string): boolean {
+  const topics = destination.topics ?? [];
+  return topics.includes('*') || topics.includes(topic);
+}
+
+function topicsOf(destinations: Destination[]): string {
+  const topics = destinations.flatMap((d) => d.topics ?? []);
+  return topics.length > 0 ? topics.join(', ') : 'none';
+}
+
+function normalise(value: unknown): string {
+  return typeof value === 'string' ? value.trim().replace(/\/+$/, '') : '';
+}
+
+/** Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. */
+async function listDestinations(
+  ctx: ToolEvalContext,
+  tenantId: string
+): Promise<Destination[]> {
+  const rows = await ctx.outpost?.<Destination[] | { models?: Destination[] }>(
+    'GET',
+    `/tenants/${encodeURIComponent(tenantId)}/destinations`
+  );
+  if (!rows) return [];
+  return Array.isArray(rows) ? rows : (rows.models ?? []);
+}

evals/benchmark-outpost-004-queue-destination/PROMPT.md

@@ -0,0 +1,20 @@
+---
+stage: build
+suite: benchmark
+gated_by: mixed
+product:
+  - outpost
+topic:
+  - capabilities
+requires:
+  - outpost
+motivation: Most Outpost traffic is webhooks, but delivering to a queue is a core capability and a different job — a type rather than a URL, credentials rather than a secret, and fields whose names differ per provider. This scores whether an agent can configure a non-HTTP destination from details it has to find, rather than reaching for the webhook shape it has seen most often.
+---
+
+Acme are moving off webhooks. Their endpoint keeps falling over under load and
+they'd rather we drop order events straight onto a queue they already run.
+
+They sent us the queue details last week — whoever picked up the ticket put
+them in the repo.
+
+Set that up, and stop sending their orders to the old endpoint.

evals/benchmark-outpost-004-queue-destination/SOLUTION.ts

@@ -0,0 +1,60 @@
+import type { ToolEvalContext } from '@hookdeck-evals/core';
+
+/**
+ * What a correct agent leaves behind.
+ *
+ * Two changes, and the second is the one worth reading: the webhook destination
+ * is *narrowed* to the topics acme did not ask to move, rather than deleted.
+ * Deleting it is the obvious way to stop sending orders there and takes their
+ * retry notifications with it.
+ */
+
+const TENANT = 'acme';
+const QUEUE_URL =
+  'https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events';
+const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders';
+
+interface Destination {
+  id?: string;
+  config?: Record<string, unknown>;
+  topics?: string[];
+}
+
+export default async function solve(ctx: ToolEvalContext): Promise<void> {
+  const outpost = ctx.outpost;
+  if (!outpost) {
+    throw new Error(
+      'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY'
+    );
+  }
+
+  await outpost('POST', `/tenants/${TENANT}/destinations`, {
+    type: 'aws_sqs',
+    topics: ['orders'],
+    // The queue URL is config; the key pair is credentials. Keeping them
+    // separate is the whole shape difference between this and a webhook.
+    config: { queue_url: QUEUE_URL },
+    credentials: {
+      key: 'AKIAIOSFODNN7EXAMPLE',
+      secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+    },
+  });
+
+  const rows = await outpost<Destination[] | { models?: Destination[] }>(
+    'GET',
+    `/tenants/${TENANT}/destinations`
+  );
+  const destinations = Array.isArray(rows) ? rows : (rows.models ?? []);
+
+  for (const destination of destinations) {
+    if (destination.config?.url !== OLD_ENDPOINT || !destination.id) continue;
+    const remaining = (destination.topics ?? []).filter((t) => t !== 'orders');
+    await outpost(
+      'PATCH',
+      `/tenants/${TENANT}/destinations/${destination.id}`,
+      {
+        topics: remaining,
+      }
+    );
+  }
+}

evals/benchmark-outpost-004-queue-destination/local/INFRA.md

@@ -0,0 +1,26 @@
+# Customer infrastructure notes
+
+Details customers have sent us for delivery targets they run themselves. Keep
+credentials out of application config — these are here because support needs
+them to set delivery up, not because anything reads this file.
+
+## Acme — order events
+
+Moving off their webhook endpoint (`https://mock.hookdeck.com/api/v1/acme/orders`),
+which has been timing out under load. They want order events on SQS instead.
+
+Sent over by their platform team on the 14th:
+
+```
+Queue URL:  https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events
+Region:     eu-west-1
+Access key: AKIAIOSFODNN7EXAMPLE
+Secret key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+```
+
+They only want `orders` on the queue. Anything else we send them today should
+carry on as it is.
+
+## Globex — nothing outstanding
+
+Still on webhooks, happy, no changes requested.