From 561e1cfc4b9ff84279b59559edb0d476c7cfeb36 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Tue, 22 Sep 2026 20:54:32 -0700 Subject: [PATCH 01/16] Record unfinished deploy status checks as incomplete `clerk deploy status` exits 1 on purpose when a deploy is not finished, so scripts can gate on it. Telemetry read that nonzero exit as a failure, which made most of the "deploy errors" series a status check reporting "not done yet" rather than anything going wrong. - Add `incomplete` to the telemetry outcome values, and let a command declare what its own soft exit meant instead of having the program infer "nonzero, therefore error". The exit code is a per-command transport detail, so only the command knows what its nonzero exit meant; an undeclared soft exit is still recorded as an error. - Declare `incomplete` from `clerk deploy status` on the one branch where the report is built and not complete. Output, exit codes, and the error path are unchanged. - Fix the payload shape once, ahead of the milestones that fill it: add `pause_step` and a nested `components` object, both sent as null, plus the five deploy report states on `stage`. Null means never observed and must never be read as false. - Document the deploy outcome semantics in the command README, including that a successful command is not a finished deploy. The warehouse classification matching both the old and new row shapes shipped first, so no chart moves when this releases. --- .changeset/grow-1233-cli-deploy-telemetry.md | 5 + packages/cli-core/src/cli-program.ts | 10 +- .../cli-core/src/commands/deploy/README.md | 6 + .../commands/deploy/status-command.test.ts | 110 ++++++++++++ .../src/commands/deploy/status-command.ts | 7 + packages/cli-core/src/lib/telemetry.test.ts | 167 +++++++++++++++--- packages/cli-core/src/lib/telemetry.ts | 120 ++++++++++++- 7 files changed, 395 insertions(+), 30 deletions(-) create mode 100644 .changeset/grow-1233-cli-deploy-telemetry.md diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md new file mode 100644 index 000000000..a830dc875 --- /dev/null +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -0,0 +1,5 @@ +--- +"clerk": patch +--- + +Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry. Output and exit codes are unchanged. diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index ad53816fc..238e7be65 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -51,6 +51,7 @@ import { finalizeAndSendTelemetry, startCommandTelemetry, telemetryResultForError, + telemetryResultForSoftExit, } from "./lib/telemetry.ts"; /** @@ -246,12 +247,11 @@ export async function runProgram( // the exit for that case; racing it here would report the wrong outcome. if (interruptedExitCode() !== null) return; // Some commands report failure via process.exitCode instead of throwing — - // read it back so telemetry doesn't record them as successes. + // read it back so telemetry doesn't record them as successes. What a + // nonzero code there *meant* is the command's to say, via + // `declareSoftExitOutcome`; absent a declaration this is still an error. const softExitCode = Number(process.exitCode ?? EXIT_CODE.SUCCESS); - await finalizeAndSendTelemetry({ - outcome: softExitCode === EXIT_CODE.SUCCESS ? "success" : "error", - exitCode: softExitCode, - }); + await finalizeAndSendTelemetry(telemetryResultForSoftExit(softExitCode)); } catch (error) { if (interruptedExitCode() !== null) return; // Started before rendering so the message is printed before we block on the diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 4c97feea8..c1f3f8f4f 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -62,6 +62,12 @@ Exit codes: | `1` | The check ran successfully, but deploy is incomplete. Inspect `state` and `nextAction`. | | else | A real CLI error occurred, such as not linked or an API failure, via the standard error path. | +### What telemetry records about a deploy run + +Telemetry's `outcome` says what happened to the _command_, not to the deploy. A `clerk deploy status` run on a deploy that is not finished records `outcome: "incomplete"` rather than `"error"`: the check ran and answered, and nothing failed. It still exits 1, so `clerk deploy status && ./cutover.sh` stops as before, and it carries no error code, because nothing was thrown. A run that fails for a real reason — not linked, an API error — throws and is recorded as an error with that error's code, unchanged. + +`success` does not mean the deploy is finished either. `clerk deploy` under an agent prints a status report and exits 0 even when no production instance exists. How far a deploy got is carried by the `stage` and `components` payload fields, never by `outcome`. + Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: 1. `--mode` CLI flag diff --git a/packages/cli-core/src/commands/deploy/status-command.test.ts b/packages/cli-core/src/commands/deploy/status-command.test.ts index 72836d9c3..f96640182 100644 --- a/packages/cli-core/src/commands/deploy/status-command.test.ts +++ b/packages/cli-core/src/commands/deploy/status-command.test.ts @@ -24,6 +24,23 @@ const { _setConfigDir, setProfile } = await import("../../lib/config.ts"); const { setMode } = await import("../../mode.ts"); const { beginInterrupt, _resetInterruptState } = await import("../../lib/signals.ts"); const { deployStatus, humanNextAction } = await import("./status-command.ts"); +const { startCommandTelemetry, telemetryResultForSoftExit } = + await import("../../lib/telemetry.ts"); + +/** A telemetry context to declare into — `deploy status` under the real program. */ +function fakeDeployStatusCommand() { + return { + name: () => "status", + options: [], + getOptionValueSource: () => undefined, + parent: { + name: () => "deploy", + options: [], + getOptionValueSource: () => undefined, + parent: null, + }, + }; +} /** What an in-flight request rejects with once Ctrl-C aborts the shared signal. */ function abortError(): Error { @@ -696,6 +713,99 @@ describe("deploy status", () => { // No records are outstanding, so no records block. expect(output).not.toContain("Add the following records"); }); + + // An unfinished deploy is not a failed command. What telemetry records is + // read back through the same soft-exit path `runProgram` uses, so these pin + // the recorded event rather than the setter call. + describe("telemetry", () => { + function recordedResult() { + return telemetryResultForSoftExit(Number(process.exitCode ?? EXIT_CODE.SUCCESS)); + } + + test("a deploy with no production instance is incomplete, not an error", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(false)); + + await deployStatus(); + + expect(process.exitCode).toBe(EXIT_CODE.GENERAL); + expect(recordedResult()).toEqual({ outcome: "incomplete", exitCode: EXIT_CODE.GENERAL }); + }); + + test("a provisioning domain is incomplete", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(true)); + mockListApplicationDomains.mockResolvedValue({ data: [], total_count: 0 }); + + await deployStatus(); + + expect(recordedResult()).toEqual({ outcome: "incomplete", exitCode: EXIT_CODE.GENERAL }); + }); + + test("a deploy still waiting on DNS is incomplete", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(pendingDnsDomainStatus()); + + await deployStatus(); + + expect(recordedResult()).toEqual({ outcome: "incomplete", exitCode: EXIT_CODE.GENERAL }); + }); + + test("a verified domain still missing OAuth credentials is incomplete", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockFetchInstanceConfig.mockImplementation(() => ({ + connection_oauth_google: { enabled: true }, + })); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(completeDomainStatus()); + + await deployStatus(); + + expect(recordedResult()).toEqual({ outcome: "incomplete", exitCode: EXIT_CODE.GENERAL }); + }); + + test("a complete deploy declares nothing and is a success at exit 0", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(completeDomainStatus()); + + await deployStatus(); + + expect(process.exitCode).toBe(EXIT_CODE.SUCCESS); + expect(recordedResult()).toEqual({ outcome: "success", exitCode: EXIT_CODE.SUCCESS }); + }); + + // A throw goes to `telemetryResultForError`, not the soft-exit branch, so + // what matters is that the run left no declaration behind: were one to + // survive a failure, the next reader of the soft exit would call it + // "incomplete" and the failure would leave the error series. + test("an API failure before the report declares nothing", async () => { + startCommandTelemetry(fakeDeployStatusCommand()); + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + + await expect(deployStatus()).rejects.toBeInstanceOf(PlapiError); + + expect(telemetryResultForSoftExit(EXIT_CODE.GENERAL)).toEqual({ + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + }); + }); + }); }); async function routePlapiFetch( diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index 39705c6bd..6921f4a88 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -4,6 +4,7 @@ import { log } from "../../lib/log.ts"; import { interruptedExitCode } from "../../lib/signals.ts"; import { sleep } from "../../lib/sleep.ts"; import { withSpinner } from "../../lib/spinner.ts"; +import { declareSoftExitOutcome } from "../../lib/telemetry.ts"; import { deployComponentLabels, dnsRecords, type DeployComponentStatus } from "./copy.ts"; import { buildDeployStatusReport, @@ -68,6 +69,12 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { return { name: () => "list", options: [], getOptionValueSource: () => undefined, parent: null }; } + /** Captures the payload of the single event a finalize call sends. */ + async function sendAndCapturePayload( + run: () => void | Promise, + result: TelemetryResult | (() => TelemetryResult), + ): Promise> { + await markTelemetryNoticeShown(); // past the grace run — reach the send path + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + let sent: string | undefined; + globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { + sent = init.body; + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeCommand()); + await run(); + // Resolved after `run` so a result derived from context (the soft-exit + // declaration) sees what the run declared. + await finalizeAndSendTelemetry(typeof result === "function" ? result() : result); + + expect(sent).toBeDefined(); + const parsed = JSON.parse(sent as string) as { + events: { payload: Record }[]; + }; + return parsed.events[0]!.payload; + } + test("no-op when telemetry is disabled (no fetch, no throw)", async () => { let called = 0; globalThis.fetch = (async () => { @@ -461,30 +489,6 @@ describe("finalizeAndSendTelemetry", () => { }); describe("stage", () => { - /** Captures the payload of the single event a finalize call sends. */ - async function sendAndCapturePayload( - run: () => void | Promise, - result: TelemetryResult, - ): Promise> { - await markTelemetryNoticeShown(); // past the grace run — reach the send path - process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; - let sent: string | undefined; - globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { - sent = init.body; - return new Response("{}"); - }) as unknown as typeof fetch; - - startCommandTelemetry(fakeCommand()); - await run(); - await finalizeAndSendTelemetry(result); - - expect(sent).toBeDefined(); - const parsed = JSON.parse(sent as string) as { - events: { payload: Record }[]; - }; - return parsed.events[0]!.payload; - } - test("reports the furthest stage reached on success", async () => { const payload = await sendAndCapturePayload( () => { @@ -529,4 +533,119 @@ describe("finalizeAndSendTelemetry", () => { expect(() => setTelemetryStage("flags")).not.toThrow(); }); }); + + // A command that reports failure through `process.exitCode` never reaches + // `telemetryResultForError`, so without a declaration the only thing the + // soft-exit branch can say is "nonzero, therefore error". + describe("soft-exit declarations", () => { + test("a declared outcome is recorded when the exit code is nonzero", async () => { + const payload = await sendAndCapturePayload( + () => declareSoftExitOutcome("incomplete"), + () => telemetryResultForSoftExit(EXIT_CODE.GENERAL), + ); + expect(payload.outcome).toBe("incomplete"); + expect(payload.exit_code).toBe(EXIT_CODE.GENERAL); + expect(payload.error_code).toBeNull(); + }); + + // The declaration says what a *failure* meant. A run that ended at 0 did + // not fail, and honoring a stale declaration would invent one. + test("a declared outcome is ignored when the run exits 0", async () => { + const payload = await sendAndCapturePayload( + () => declareSoftExitOutcome("incomplete"), + () => telemetryResultForSoftExit(EXIT_CODE.SUCCESS), + ); + expect(payload.outcome).toBe("success"); + }); + + test("an undeclared nonzero soft exit is still an error", async () => { + const payload = await sendAndCapturePayload( + () => {}, + () => telemetryResultForSoftExit(EXIT_CODE.GENERAL), + ); + expect(payload.outcome).toBe("error"); + expect(payload.error_code).toBeNull(); + }); + + // The shape M7 extends: `clerk api` holds a code its own catch swallowed. + test("a declaration can carry an error code", () => { + startCommandTelemetry(fakeCommand()); + declareSoftExitOutcome("error", "api_not_found"); + expect(telemetryResultForSoftExit(EXIT_CODE.GENERAL)).toEqual({ + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + errorCode: "api_not_found", + }); + }); + + // A thrown error is the more specific fact, and `runProgram` routes it + // through the other classifier entirely. + test("a thrown error keeps its own code regardless of a declaration", async () => { + const payload = await sendAndCapturePayload( + () => declareSoftExitOutcome("incomplete"), + telemetryResultForError(new CliError("boom", { code: ERROR_CODE.NOT_LINKED })), + ); + expect(payload.outcome).toBe("error"); + expect(payload.error_code).toBe("not_linked"); + }); + + // Tests share the module, and so would two runs in one process: a stale + // declaration would relabel the next command's failure. + test("a declaration does not leak into the next run", () => { + startCommandTelemetry(fakeCommand()); + declareSoftExitOutcome("incomplete"); + startCommandTelemetry(fakeCommand()); + expect(telemetryResultForSoftExit(EXIT_CODE.GENERAL)).toEqual({ + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + }); + }); + + test("declaring with no active context is a no-op", () => { + expect(() => declareSoftExitOutcome("incomplete")).not.toThrow(); + }); + }); + + // The warehouse parses this payload by key. A rename splits a column in two + // without failing anything here, so the key set itself is the contract. + describe("payload shape", () => { + test("carries exactly the agreed keys", async () => { + const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); + expect(Object.keys(payload).sort()).toEqual( + [ + "ai_agent", + "app_id", + "arch", + "ci", + "command", + "components", + "duration_ms", + "env", + "error_code", + "exit_code", + "flags", + "in_screen", + "in_tmux", + "install_method", + "machine_uuid", + "mode", + "os", + "outcome", + "pause_step", + "stage", + "terminal_program", + "workspace_id", + ].sort(), + ); + }); + + // Declared in M3 so the shape is fixed once; the wizard fills them in + // later milestones. Null means never observed, and the warehouse reads it + // that way — it must not arrive as `false` or as an absent key. + test("the fields later milestones fill are present and null", async () => { + const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); + expect(payload.pause_step).toBeNull(); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + }); }); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 5880a435d..2bd76c6b4 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -36,12 +36,47 @@ import { log } from "./log.ts"; import { getMode } from "../mode.ts"; import { CURRENT_VERSION, IS_DEV_BUILD } from "./version.ts"; +/** + * What happened to the command, not to the thing it acted on. + * + * `incomplete` says the command ran and the thing it reports on is not + * finished — nobody is being asked to do anything, and nothing failed. Only + * `clerk deploy status` sends it, and only by declaring it (see + * {@link declareSoftExitOutcome}); it is never a mapping of nonzero exits. + * + * `success` is not "the deploy is done" either: `clerk deploy` under an agent + * prints a status report and exits 0 with nothing started. How far a deploy + * got is `stage` and `components`, never `outcome`. + */ +export type TelemetryOutcome = "success" | "error" | "abort" | "incomplete"; + export type TelemetryResult = { - outcome: "success" | "error" | "abort"; + outcome: TelemetryOutcome; exitCode: number; errorCode?: string; }; +/** + * Where a `clerk deploy` run stopped when the user has something left to do. + * Narrower than `stage`: on a fresh deploy the DNS handoff runs before OAuth + * setup, so someone who skips a provider is at `stage: "domain_pending"` and + * `pauseStep: "oauth"`. Set by the deploy wizard (GROW-1233 item 2). + */ +export type TelemetryPauseStep = "dns" | "oauth"; + +/** + * Per-component readiness at the time the run ended. `null` means never + * observed — no successful status read established it — and must never be + * read as `false`: a failed status call is not a DNS failure. Filled by the + * deploy wizard and `clerk deploy status` (GROW-1233 item 4). + */ +export type TelemetryComponents = { + dns: boolean | null; + ssl: boolean | null; + mail: boolean | null; + oauth: boolean | null; +}; + /** * Closed set of drop-off points a command can report. A union rather than a * bare string so a typo or a rename that misses a call site fails to compile @@ -70,6 +105,20 @@ export type TelemetryStage = | "token_exchange" | "store" | "first_application" + // `clerk deploy` and `clerk deploy status` + // + // Unlike the groups above, these are not control-flow positions: each is a + // state of the deploy itself, as `resolveActiveReportState` in + // `commands/deploy/status.ts` would compute it at that moment. So the stage + // a wizard run reports and the stage `clerk deploy status` reports a second + // later agree about the same deploy. The last one set is sent, and a run + // that ends before any state resolves sends null rather than defaulting — + // "never established" is a distinct answer from "not started". + | "not_started" + | "domain_provisioning" + | "domain_pending" + | "oauth_pending" + | "complete" // shared terminal marker | "done"; @@ -87,8 +136,29 @@ type TelemetryContext = { startedAt: number; /** Last stage set — see setTelemetryStage. */ stage: TelemetryStage | null; + /** Declared by the command for the soft-exit path — see declareSoftExitOutcome. */ + softExit: SoftExitDeclaration | null; + /** Where the deploy wizard stopped — see TelemetryPauseStep. */ + pauseStep: TelemetryPauseStep | null; + /** Per-component readiness, each field written only by an observation. */ + components: TelemetryComponents; }; +/** + * What a command wants recorded when it reports failure through + * `process.exitCode` rather than by throwing. The error code is optional + * because `clerk deploy status` has none to give: nothing was thrown, so + * there is no code, and `incomplete` is the whole answer. + */ +type SoftExitDeclaration = { + outcome: TelemetryOutcome; + errorCode?: string; +}; + +function emptyComponents(): TelemetryComponents { + return { dns: null, ssl: null, mail: null, oauth: null }; +} + let context: TelemetryContext | null = null; /** @@ -186,6 +256,9 @@ export function startCommandTelemetry(actionCommand: TelemetryCommand): void { flags: collectSetFlagNames(actionCommand).join(","), startedAt: Date.now(), stage: null, + softExit: null, + pauseStep: null, + components: emptyComponents(), }; } catch (error) { log.debug(`telemetry: failed to start context: ${error}`); @@ -208,6 +281,46 @@ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; } +/** + * Declare what this run should be recorded as when it ends by setting + * `process.exitCode` instead of throwing. + * + * Commands that catch their own failure never reach `telemetryResultForError`, + * so without this the soft-exit branch in `cli-program.ts` can only say + * "nonzero, therefore error". That is wrong in both directions: `clerk deploy + * status` exits 1 on a deploy that simply is not finished, and `clerk api` + * exits 1 holding an error code it never gets to record. + * + * Why this is a declaration and not a rule about exit codes: the exit code is + * a per-command transport detail — 1 means "not done" from `deploy status` + * and "request failed" from `api` — so only the command knows what its own + * nonzero exit meant. A general mapping would relabel every command at once. + * + * Ignored when the run throws: a thrown error is the more specific fact, and + * `runProgram` classifies it through {@link telemetryResultForError}. + */ +export function declareSoftExitOutcome(outcome: TelemetryOutcome, errorCode?: string): void { + if (context) context.softExit = { outcome, errorCode }; +} + +/** + * How a run that set `process.exitCode` and returned is recorded. Honors a + * declaration only on a nonzero exit: a command that declared an outcome and + * then succeeded anyway (a retry that worked, a later branch clearing the + * code) is a success, and reporting the stale declaration would invent a + * failure the user never saw. + */ +export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { + if (exitCode === EXIT_CODE.SUCCESS) return { outcome: "success", exitCode }; + const declared = context?.softExit; + if (!declared) return { outcome: "error", exitCode }; + return { + outcome: declared.outcome, + exitCode, + ...(declared.errorCode ? { errorCode: declared.errorCode } : {}), + }; +} + export function telemetryResultForError(error: unknown): TelemetryResult { if (error instanceof UserAbortError) { return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; @@ -297,6 +410,11 @@ async function buildAndSend( exit_code: result.exitCode, error_code: result.errorCode ?? null, stage: current.stage, + pause_step: current.pauseStep, + // Nested rather than four flat keys: it is one JSON path per component + // in the warehouse, and the group is obviously one thing. A null member + // means never observed — see TelemetryComponents. + components: current.components, duration_ms: Date.now() - current.startedAt, machine_uuid: machineUuid, install_method: detectInstallMethod(process.env, process.execPath), From 9cb8d944ee477cbbb9bbf4680fe9b5904d9e93bb Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Tue, 22 Sep 2026 20:56:54 -0700 Subject: [PATCH 02/16] Make the soft-exit declaration unable to contradict itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the `incomplete` outcome. No behavior change: same output, same exit codes, same recorded outcomes. - Narrow what a command may declare for its own soft exit to `incomplete` or `error`. `success` was accepted, and declaring it on a run that then exits nonzero produces a row the warehouse reads as a success — its classifier tests `outcome = 'success'` ahead of every error rule and never reads `exit_code` — so the failure would leave the error series with nothing able to reconcile it. `abort` is excluded because the interrupt path reports itself. Three more call sites arrive in later milestones, so the type is what has to say this. - Document that the last declaration wins, and that it applies to whatever nonzero code the run ends with. A command aggregating failures across several targets and meaning to report the first must select before calling, not call from inside its loop. - Cover the whole seam end to end: an unfinished `clerk deploy status` driven through the real program emits `outcome: "incomplete"` with exit code 1 and no error code. The unit tests exercise the classifier, not `runProgram`, so an edit to the soft-exit branch could otherwise revert the milestone with the suite still green. - Record that a finished deploy's stage is `complete`, never the shared `done` marker, which the warehouse contract test rejects on deploy commands. - Note at the declaration site that only the `status` subcommand reaches it, so routing the wizard through it would make every unfinished wizard pass declare `incomplete` too. - Correct two test comments that described failures which cannot occur, and replace two hand-built command fixtures with one shared helper. --- .../commands/deploy/status-command.test.ts | 47 +++++++---------- .../src/commands/deploy/status-command.ts | 7 ++- packages/cli-core/src/lib/telemetry.test.ts | 13 +++-- packages/cli-core/src/lib/telemetry.ts | 32 +++++++++++- .../src/test/integration/telemetry.test.ts | 50 ++++++++++++++++++- packages/cli-core/src/test/lib/stubs.ts | 19 +++++++ 6 files changed, 128 insertions(+), 40 deletions(-) diff --git a/packages/cli-core/src/commands/deploy/status-command.test.ts b/packages/cli-core/src/commands/deploy/status-command.test.ts index f96640182..9a2036f6b 100644 --- a/packages/cli-core/src/commands/deploy/status-command.test.ts +++ b/packages/cli-core/src/commands/deploy/status-command.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { EXIT_CODE, PlapiError } from "../../lib/errors.ts"; -import { stubFetch, useCaptureLog } from "../../test/lib/stubs.ts"; +import { fakeTelemetryCommand, stubFetch, useCaptureLog } from "../../test/lib/stubs.ts"; const mockFetchApplication = mock(); const mockListApplicationDomains = mock(); @@ -27,21 +27,6 @@ const { deployStatus, humanNextAction } = await import("./status-command.ts"); const { startCommandTelemetry, telemetryResultForSoftExit } = await import("../../lib/telemetry.ts"); -/** A telemetry context to declare into — `deploy status` under the real program. */ -function fakeDeployStatusCommand() { - return { - name: () => "status", - options: [], - getOptionValueSource: () => undefined, - parent: { - name: () => "deploy", - options: [], - getOptionValueSource: () => undefined, - parent: null, - }, - }; -} - /** What an in-flight request rejects with once Ctrl-C aborts the shared signal. */ function abortError(): Error { return new DOMException("The operation was aborted.", "AbortError"); @@ -714,16 +699,18 @@ describe("deploy status", () => { expect(output).not.toContain("Add the following records"); }); - // An unfinished deploy is not a failed command. What telemetry records is - // read back through the same soft-exit path `runProgram` uses, so these pin - // the recorded event rather than the setter call. + // An unfinished deploy is not a failed command. These read the declaration + // back through the classifier the soft-exit branch calls, so they pin what + // would be recorded rather than that the setter ran. The whole path, + // including `runProgram` itself, is covered end to end in + // `test/integration/telemetry.test.ts`. describe("telemetry", () => { function recordedResult() { return telemetryResultForSoftExit(Number(process.exitCode ?? EXIT_CODE.SUCCESS)); } test("a deploy with no production instance is incomplete, not an error", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(false)); await deployStatus(); @@ -733,7 +720,7 @@ describe("deploy status", () => { }); test("a provisioning domain is incomplete", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(true)); mockListApplicationDomains.mockResolvedValue({ data: [], total_count: 0 }); @@ -743,7 +730,7 @@ describe("deploy status", () => { }); test("a deploy still waiting on DNS is incomplete", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(true)); mockDomain(); mockOAuthComplete(); @@ -756,7 +743,7 @@ describe("deploy status", () => { }); test("a verified domain still missing OAuth credentials is incomplete", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(true)); mockDomain(); mockOAuthComplete(); @@ -772,7 +759,7 @@ describe("deploy status", () => { }); test("a complete deploy declares nothing and is a success at exit 0", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(true)); mockDomain(); mockOAuthComplete(); @@ -785,12 +772,12 @@ describe("deploy status", () => { expect(recordedResult()).toEqual({ outcome: "success", exitCode: EXIT_CODE.SUCCESS }); }); - // A throw goes to `telemetryResultForError`, not the soft-exit branch, so - // what matters is that the run left no declaration behind: were one to - // survive a failure, the next reader of the soft exit would call it - // "incomplete" and the failure would leave the error series. - test("an API failure before the report declares nothing", async () => { - startCommandTelemetry(fakeDeployStatusCommand()); + // The declaration is made only once the report exists, never optimistically + // on the way in. Moving it above the status read would set it here and fail + // this assertion, which is the regression this guards — a run that never + // learned the deploy's state must not claim it is merely unfinished. + test("a run that fails before it has a report declares nothing", async () => { + startCommandTelemetry(fakeTelemetryCommand("deploy status")); mockFetchApplication.mockResolvedValue(appWith(true)); mockDomain(); mockOAuthComplete(); diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index 6921f4a88..3a1438911 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -72,8 +72,11 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { }); function fakeCommand(): TelemetryCommand { - return { name: () => "list", options: [], getOptionValueSource: () => undefined, parent: null }; + return fakeTelemetryCommand("list"); } /** Captures the payload of the single event a finalize call sends. */ @@ -578,12 +578,15 @@ describe("finalizeAndSendTelemetry", () => { }); }); - // A thrown error is the more specific fact, and `runProgram` routes it - // through the other classifier entirely. + // What the send does with a result it is handed while a declaration is + // live: it uses the result. Resolved through the callback so the two call + // sites read alike, and so this keeps holding if `telemetryResultForError` + // ever starts reading context — it is pure today, so the ordering itself + // makes no difference. test("a thrown error keeps its own code regardless of a declaration", async () => { const payload = await sendAndCapturePayload( () => declareSoftExitOutcome("incomplete"), - telemetryResultForError(new CliError("boom", { code: ERROR_CODE.NOT_LINKED })), + () => telemetryResultForError(new CliError("boom", { code: ERROR_CODE.NOT_LINKED })), ); expect(payload.outcome).toBe("error"); expect(payload.error_code).toBe("not_linked"); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 2bd76c6b4..9c8716610 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -50,6 +50,18 @@ import { CURRENT_VERSION, IS_DEV_BUILD } from "./version.ts"; */ export type TelemetryOutcome = "success" | "error" | "abort" | "incomplete"; +/** + * What a command may declare for itself on the soft-exit path. + * + * Deliberately narrower than {@link TelemetryOutcome}. `success` is excluded + * because declaring it on a run that then exits nonzero produces a row the + * warehouse reads as a success — its classifier tests `outcome = 'success'` + * ahead of every error rule and never reads `exit_code` — so the failure + * would leave the error series with nothing able to reconcile it. `abort` is + * excluded because it belongs to the interrupt path, which reports itself. + */ +export type SoftExitOutcome = "incomplete" | "error"; + export type TelemetryResult = { outcome: TelemetryOutcome; exitCode: number; @@ -114,6 +126,11 @@ export type TelemetryStage = // later agree about the same deploy. The last one set is sent, and a run // that ends before any state resolves sends null rather than defaulting — // "never established" is a distinct answer from "not started". + // + // A finished deploy is `complete`, never the shared `done` marker below: + // the warehouse's payload contract test accepts exactly these five values + // on `deploy run` and `deploy status`, so `done` there trips it on every + // finished deploy. | "not_started" | "domain_provisioning" | "domain_pending" @@ -151,7 +168,7 @@ type TelemetryContext = { * there is no code, and `incomplete` is the whole answer. */ type SoftExitDeclaration = { - outcome: TelemetryOutcome; + outcome: SoftExitOutcome; errorCode?: string; }; @@ -298,8 +315,19 @@ export function currentTelemetryStage(): TelemetryStage | null { * * Ignored when the run throws: a thrown error is the more specific fact, and * `runProgram` classifies it through {@link telemetryResultForError}. + * + * Two rules for callers: + * + * - **The last call wins.** Call this once, with the fact you want recorded. + * A command that aggregates failures across several targets and means to + * report the first one must select that error before calling, not call from + * inside its loop — which would record the last target's failure instead, + * with no test failing and telemetry naming the wrong thing. + * - **It applies to whatever nonzero code the run ends with,** not only the + * one in force when it was called. Declare it under the same condition that + * sets the exit code, so the two cannot diverge. */ -export function declareSoftExitOutcome(outcome: TelemetryOutcome, errorCode?: string): void { +export function declareSoftExitOutcome(outcome: SoftExitOutcome, errorCode?: string): void { if (context) context.softExit = { outcome, errorCode }; } diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts index 691962126..07abac7b5 100644 --- a/packages/cli-core/src/test/integration/telemetry.test.ts +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -5,7 +5,14 @@ */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { clerk, http, useIntegrationTestHarness } from "./lib/harness.ts"; +import { + clerk, + getInstance, + http, + MOCK_APP_DEV_ONLY, + setProfile, + useIntegrationTestHarness, +} from "./lib/harness.ts"; import { useCaptureLog } from "../lib/stubs.ts"; useIntegrationTestHarness(); @@ -116,6 +123,47 @@ test("records failures with error code and reuses the machine uuid", async () => expect(event.payload.machine_uuid).toBe(firstUuid); }); +// The counterpart to the soft-failure test below: `clerk deploy status` exits +// 1 on a deploy that is merely unfinished, and declares what that 1 meant. +// Driven through the real program so the whole seam is covered — the +// declaration, the soft-exit branch reading it back, and the emitted event — +// which no unit test of the classifier can do on its own. +test("an unfinished `deploy status` is recorded as incomplete, not an error", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + await setProfile("github.com/test/project", { + workspaceId: "", + appId: MOCK_APP_DEV_ONLY.application_id, + instances: { development: getInstance(MOCK_APP_DEV_ONLY, "development").instance_id }, + }); + // Development-only: no production instance, so the report is `not_started` + // and no domain or config call follows. + http.mock({ + [`/applications/${MOCK_APP_DEV_ONLY.application_id}`]: MOCK_APP_DEV_ONLY, + "test-telemetry.clerk.com": {}, + }); + + try { + await clerk.raw("deploy", "status"); + // Set by the command rather than thrown, so the harness's own result + // reports 0 — the soft exit is on the process, which is what the run + // would exit with and what the event has to carry. + expect(process.exitCode).toBe(1); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("deploy status"); + expect(event.payload.outcome).toBe("incomplete"); + expect(event.payload.exit_code).toBe(1); + // Nothing was thrown, so there is no code to carry — the two fields are + // unrelated, and `incomplete` is the whole answer. + expect(event.payload.error_code).toBeNull(); + } finally { + process.exitCode = undefined; + } +}); + test("maps a soft failure (process.exitCode set without throwing) to outcome error", async () => { await markNoticeAlreadyShown(); process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index b45e49c42..57dc7330d 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -2,6 +2,7 @@ import { Writable } from "node:stream"; import { afterEach, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; +import type { TelemetryCommand } from "../../lib/telemetry.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -243,3 +244,21 @@ type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise< export function stubFetch(impl: FetchImpl): void { globalThis.fetch = impl as typeof fetch; } + +/** + * A stand-in for the Commander command telemetry reads, built from a space + * separated command path: `"deploy status"` yields a `status` command whose + * parent is `deploy`, which is what `startCommandTelemetry` walks to produce + * the payload's `command` field. No flags are reported as set. + */ +export function fakeTelemetryCommand(path: string): TelemetryCommand { + const noOptions = { options: [] as never[], getOptionValueSource: () => undefined }; + // Root first, so each command's parent is the segment to its left. The + // outermost parent is null: telemetry excludes the root `clerk` itself. + return path + .split(" ") + .reduce( + (parent, segment) => ({ name: () => segment, ...noOptions, parent }), + null, + ) as TelemetryCommand; +} From f194a54da8dcfc66da205f9d906bc509095395e1 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Tue, 22 Sep 2026 22:08:25 -0700 Subject: [PATCH 03/16] Give deploy endings and doctor crashes error codes A skipped OAuth provider, an interrupted prompt, and a wait on Clerk's provisioning all left `clerk deploy` recorded as `cli_error`. `deployPausedError` now takes one reason that selects the error code and exit code together: `deploy_paused` at exit 1, `deploy_cancelled` at exit 130, `deploy_finalizing` at exit 1. The same call records `pause_step` (`dns` or `oauth`), except on the finalizing wait, where nobody stopped at a step. Exit codes and printed output are unchanged. The wizard's two "production instance could not be resolved" throws send `deploy_instance_unresolved` instead of `usage_error`; exit code stays 2. `clerk doctor` now names the check that threw instead of reporting an anonymous crash, marks the result `crashed: true`, and throws `doctor_check_crashed` rather than `doctor_failed` so a CLI bug is distinguishable from a real finding. Check names come from one `CHECK_NAME` map so the registry and the checks cannot disagree. Tests assert the posted telemetry payload for each ending, both instance-unresolved sites, and the three doctor outcomes. --- .changeset/grow-1233-cli-deploy-telemetry.md | 4 +- .../cli-core/src/commands/deploy/README.md | 10 + .../src/commands/deploy/index.test.ts | 251 +++++++++++++++++- .../cli-core/src/commands/deploy/index.ts | 14 +- .../cli-core/src/commands/deploy/state.ts | 50 +++- .../cli-core/src/commands/doctor/README.md | 18 +- .../cli-core/src/commands/doctor/check-mcp.ts | 9 +- .../cli-core/src/commands/doctor/checks.ts | 42 ++- .../src/commands/doctor/index.test.ts | 125 +++++++++ .../cli-core/src/commands/doctor/index.ts | 70 +++-- .../cli-core/src/commands/doctor/types.ts | 7 + packages/cli-core/src/lib/errors.ts | 22 ++ packages/cli-core/src/lib/telemetry.test.ts | 24 ++ packages/cli-core/src/lib/telemetry.ts | 14 + .../src/test/integration/telemetry.test.ts | 19 ++ 15 files changed, 628 insertions(+), 51 deletions(-) create mode 100644 packages/cli-core/src/commands/doctor/index.test.ts diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index a830dc875..a4006dcf2 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -2,4 +2,6 @@ "clerk": patch --- -Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry. Output and exit codes are unchanged. +Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Output and exit codes are unchanged. + +`clerk doctor` now names the check that crashed instead of reporting "Unknown check", and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index c1f3f8f4f..95efd592c 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -68,6 +68,16 @@ Telemetry's `outcome` says what happened to the _command_, not to the deploy. A `success` does not mean the deploy is finished either. `clerk deploy` under an agent prints a status report and exits 0 even when no production instance exists. How far a deploy got is carried by the `stage` and `components` payload fields, never by `outcome`. +The wizard's three non-crash endings all exit the way they always have, and are told apart by their error code rather than by `exit_code` — which cannot separate the first from the third, since both are 1. + +| Ending | `error_code` | `pause_step` | Exit | +| ------------------------------------------------------------------- | ------------------- | ------------------------ | ---- | +| The user skipped an OAuth provider or a DNS check | `deploy_paused` | the step they stopped on | 1 | +| The user interrupted a prompt after the production instance existed | `deploy_cancelled` | the step they stopped on | 130 | +| Every DNS component verified, Clerk still provisioning | `deploy_finalizing` | null | 1 | + +`pause_step` is null on the last row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. A Ctrl-C _before_ the production instance exists is not any of these — there is no state to preserve, so it stays a plain `abort` at exit 0. + Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: 1. `--mode` CLI flag diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 297907195..8a0b4cc58 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -2,7 +2,7 @@ import { test, expect, describe, beforeEach, afterEach, mock, spyOn } from "bun: import { mkdtemp, rm } from "node:fs/promises"; import { join, relative } from "node:path"; import { tmpdir } from "node:os"; -import { useCaptureLog, listageStubs } from "../../test/lib/stubs.ts"; +import { useCaptureLog, listageStubs, fakeTelemetryCommand } from "../../test/lib/stubs.ts"; import { CliError, ERROR_CODE, EXIT_CODE, PlapiError, UserAbortError } from "../../lib/errors.ts"; const mockIsAgent = mock(); @@ -69,6 +69,8 @@ mock.module("../../lib/open.ts", () => ({ })); const { _setConfigDir, readConfig, setProfile } = await import("../../lib/config.ts"); +const { finalizeAndSendTelemetry, startCommandTelemetry, telemetryResultForError } = + await import("../../lib/telemetry.ts"); const { deploy } = await import("./index.ts"); const { providerSetupIntro, showOAuthWalkthrough } = await import("./providers.ts"); const { collectCustomDomain } = await import("./prompts.ts"); @@ -333,6 +335,22 @@ describe("deploy", () => { ); } + /** + * The Platform API answered, but the instance it returned has no id to write + * to — the one shape that reaches the wizard's "production instance could + * not be resolved" guards. + */ + function stubCreateProductionInstanceWithoutId() { + stubCreateProductionInstance(); + const withId = mockCreateProductionInstance.getMockImplementation() as ( + appId: string, + params: { domain: string }, + ) => Record; + mockCreateProductionInstance.mockImplementation( + (appId: string, params: { domain: string }) => ({ ...withId(appId, params), id: "" }), + ); + } + async function runDeployUntilPause(options: Parameters[0] = {}) { try { await runDeploy(options); @@ -2586,5 +2604,236 @@ describe("deploy", () => { const config = await readConfig(); expect(config.profiles[process.cwd()]?.instances.production).toBe("ins_prod_recovered"); }); + + // What the warehouse sees for each way a deploy run can end. These assert + // the payload rather than the thrown error, because the error code and the + // pause step travel by different routes — the code on the error, the step + // through the telemetry context — and only the payload proves both arrive + // together. + describe("what telemetry records for each ending", () => { + const TELEMETRY_URL = "https://capture.invalid/v1/event"; + let realFetch: typeof globalThis.fetch; + + beforeEach(() => { + realFetch = globalThis.fetch; + }); + afterEach(() => { + globalThis.fetch = realFetch; + delete process.env.CLERK_TELEMETRY_URL; + }); + + /** The event a `clerk deploy` run would post, plus whatever it threw. */ + async function deployTelemetry( + run: () => Promise, + ): Promise<{ payload: Record; error: CliError | undefined }> { + const { markTelemetryNoticeShown } = await import("../../lib/config.ts"); + await markTelemetryNoticeShown(); // past the grace run, which sends nothing + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + let sent: string | undefined; + globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { + sent = init.body; + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeTelemetryCommand("deploy")); + let error: CliError | undefined; + try { + await run(); + } catch (caught) { + error = caught as CliError; + } + // `runProgram` classifies a throw and reads `process.exitCode` back + // otherwise; a wizard that returns normally never sets one. + await finalizeAndSendTelemetry( + error ? telemetryResultForError(error) : { outcome: "success", exitCode: 0 }, + ); + + expect(sent).toBeDefined(); + const parsed = JSON.parse(sent as string) as { + events: { payload: Record }[]; + }; + return { payload: parsed.events[0]!.payload, error }; + } + + test("a skipped OAuth provider is a paused deploy at the oauth step", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + await runDnsHandoff(); + mockSelect.mockResolvedValueOnce("skip"); + + const { payload, error } = await deployTelemetry(async () => runDeploy({})); + + expect(error?.message).toContain("Deploy paused at: Google OAuth credential setup"); + expect(payload.outcome).toBe("error"); + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_PAUSED); + expect(payload.pause_step).toBe("oauth"); + expect(payload.exit_code).toBe(EXIT_CODE.GENERAL); + }); + + test("Ctrl-C at the OAuth prompt is a cancelled deploy at the oauth step", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + await runDnsHandoff(); + mockSelect.mockRejectedValueOnce(promptExitError()); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); + expect(payload.pause_step).toBe("oauth"); + expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); + }); + + test("Ctrl-C at the DNS retry prompt is a cancelled deploy at the dns step", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockLiveProduction({ + instanceId: "ins_prod_123", + productionConfig: { + connection_oauth_google: { + enabled: true, + client_id: "google-client-id.apps.googleusercontent.com", + client_secret: "REDACTED", + }, + }, + }); + mockGetApplicationDomainStatus.mockResolvedValue( + domainStatus({ status: "incomplete", dns: false, ssl: false, mail: false }), + ); + mockSelect.mockResolvedValueOnce("check").mockRejectedValueOnce(promptExitError()); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); + expect(payload.pause_step).toBe("dns"); + expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); + }); + + test("Ctrl-C at the BIND zone export is a cancelled deploy at the dns step", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockRejectedValueOnce(promptExitError()); + mockInput.mockResolvedValueOnce("example.com"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); + expect(payload.pause_step).toBe("dns"); + expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); + }); + + // Every DNS component passed and the user has nothing left to do, so + // this is a wait on Clerk rather than a step anyone stopped on — which + // is why it reports no pause step at all. + test("waiting on Clerk after every component verified is finalizing, at no step", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockLiveProduction({ + instanceId: "ins_prod_123", + developmentConfig: {}, + productionConfig: {}, + }); + mockGetApplicationDomainStatus.mockResolvedValue( + domainStatus({ status: "incomplete", dns: true, ssl: true, mail: true }), + ); + mockConfirm.mockResolvedValueOnce(false); + mockSelect.mockResolvedValueOnce("check"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_FINALIZING); + expect(payload.pause_step).toBeNull(); + expect(payload.exit_code).toBe(EXIT_CODE.GENERAL); + }); + + // The exit-0 endings are the control: "skip" at DNS verification is a + // finished command, and a code here would move real successes into the + // failure series. + test("choosing skip at DNS verification is a success with no code", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockFetchInstanceConfig.mockResolvedValue({}); // no OAuth providers to configure + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockSelect.mockResolvedValueOnce("skip"); + mockGetApplicationDomainStatus.mockResolvedValue( + domainStatus({ status: "incomplete", dns: false, ssl: false, mail: false }), + ); + + const { payload, error } = await deployTelemetry(async () => runDeploy({})); + + expect(error).toBeUndefined(); + expect(stripAnsi(captured.err)).toContain("Skipping DNS verification for now."); + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(0); + expect(payload.error_code).toBeNull(); + expect(payload.pause_step).toBeNull(); + }); + + // A pause is not the only way this path fails, and the pause codes must + // not swallow the ones that name a real failure. + test("a failure that is not a pause keeps its own code", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockCreateProductionInstance.mockResolvedValueOnce({ + object: "instance", + id: "ins_prod_mock", + environment_type: "production" as const, + active_domain: null, + publishable_key: "pk_live_test", + secret_key: "sk_live_test", + created_at: 1770000000000, + updated_at: 1770000000000, + }); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_DOMAIN_MISSING); + expect(payload.pause_step).toBeNull(); + }); + + // Both sites fire when the instance the wizard is about to write to + // cannot be named. That is a failure, not the malformed-input `clerk` + // was given, so it is no longer filed under `usage_error` — the exit + // code stays 2. + test("an unnameable production instance during OAuth setup is deploy_instance_unresolved", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + stubCreateProductionInstanceWithoutId(); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED); + expect(payload.exit_code).toBe(EXIT_CODE.USAGE); + }); + + test("an unnameable production instance at the next steps is deploy_instance_unresolved", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockFetchInstanceConfig.mockResolvedValue({}); // skip OAuth setup, reach finishDeploy + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockSelect.mockResolvedValueOnce("skip"); + mockGetApplicationDomainStatus.mockResolvedValue( + domainStatus({ status: "incomplete", dns: false, ssl: false, mail: false }), + ); + stubCreateProductionInstanceWithoutId(); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED); + expect(payload.exit_code).toBe(EXIT_CODE.USAGE); + }); + }); }); }); diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index 34be656bf..8b38b1471 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -439,7 +439,7 @@ async function runDnsRecordHandoff( log.blank(); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, { interrupted: true }); + throw deployPausedError(state, "cancelled"); } throw error; } @@ -478,7 +478,7 @@ async function runDnsVerificationPrompt( return await runDnsVerification(ctx, state); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, { interrupted: true }); + throw deployPausedError(state, "cancelled"); } throw error; } @@ -520,7 +520,7 @@ async function runDnsVerification( // When all DNS components are verified but the server has not yet marked the // deployment complete, the user cannot influence the remaining wait. if (outcome.status.dns && outcome.status.ssl && outcome.status.mail) { - throw deployPausedError(state); + throw deployPausedError(state, "finalizing"); } if (pendingTargets.length > 0) { @@ -533,7 +533,7 @@ async function runDnsVerification( action = await chooseDnsVerificationRetryAction(); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, { interrupted: true }); + throw deployPausedError(state, "cancelled"); } throw error; } @@ -595,6 +595,8 @@ async function runOAuthSetup( if (!productionInstanceId) { throwUsageError( "Cannot save OAuth credentials because the production instance could not be resolved. Run `clerk deploy` after confirming the production instance in the Clerk Dashboard.", + undefined, + ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED, ); } @@ -620,7 +622,7 @@ async function runOAuthSetup( pending: { type: "oauth", provider: descriptor.provider }, completedOAuthProviders: [...completed], }, - { interrupted: true }, + "cancelled", ); } throw error; @@ -706,6 +708,8 @@ async function finishDeploy( if (!productionInstanceId) { throwUsageError( "Cannot print deploy next steps because the production instance could not be resolved. Run `clerk deploy` after confirming the production instance in the Clerk Dashboard.", + undefined, + ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED, ); } await animateHeader({ diff --git a/packages/cli-core/src/commands/deploy/state.ts b/packages/cli-core/src/commands/deploy/state.ts index 49caf792c..1addfdcff 100644 --- a/packages/cli-core/src/commands/deploy/state.ts +++ b/packages/cli-core/src/commands/deploy/state.ts @@ -1,4 +1,5 @@ -import { CliError, EXIT_CODE } from "../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE, type ErrorCode } from "../../lib/errors.ts"; +import { setTelemetryPauseStep } from "../../lib/telemetry.ts"; import { pausedMessage } from "./copy.ts"; import type { CnameTarget } from "../../lib/plapi.ts"; import { providerLabel, type OAuthProvider } from "./providers.ts"; @@ -36,16 +37,47 @@ export function pausedStepDescription(state: DeployOperationState): string { export class DeployPausedError extends CliError {} /** - * Either way the deploy is unfinished, so both codes are nonzero: a production - * instance exists but DNS or OAuth is incomplete, and `clerk deploy && cutover` - * must not proceed. `interrupted` (the user stopped it) reports 130 to match - * every other Ctrl-C; otherwise it is an ordinary failure and reports 1. + * Why the deploy stopped. Three situations, not one with modifiers: a skip, an + * interrupt and a backend wait need three different follow-ups, and each gets + * its own error code so telling them apart never means reading `exit_code` + * (which cannot separate `paused` from `finalizing` — both exit 1). + * + * One argument rather than two so the reason and the exit code cannot + * disagree; the shape they share is in {@link PAUSE_REASONS}. + */ +export type DeployPauseReason = "paused" | "cancelled" | "finalizing"; + +/** + * Either way the deploy is unfinished, so every exit code here is nonzero: a + * production instance exists but DNS or OAuth is incomplete, and `clerk deploy + * && cutover` must not proceed. `cancelled` (the user stopped it) reports 130 + * to match every other Ctrl-C; the rest are ordinary failures and report 1. + */ +const PAUSE_REASONS: Record< + DeployPauseReason, + { code: ErrorCode; exitCode: typeof EXIT_CODE.GENERAL | typeof EXIT_CODE.SIGINT } +> = { + paused: { code: ERROR_CODE.DEPLOY_PAUSED, exitCode: EXIT_CODE.GENERAL }, + cancelled: { code: ERROR_CODE.DEPLOY_CANCELLED, exitCode: EXIT_CODE.SIGINT }, + finalizing: { code: ERROR_CODE.DEPLOY_FINALIZING, exitCode: EXIT_CODE.GENERAL }, +}; + +/** + * The pause every unfinished `clerk deploy` run throws, and the one place that + * records which step it stopped on — the only point that knows both, since + * `state.pending` names the step and `reason` says whether the person stopped + * there at all. A `finalizing` wait did not: the deploy is waiting on Clerk, so + * recording `dns` would count a drop-off nobody made, and the code already says + * everything the step would. + * + * The telemetry write is an in-memory assignment that cannot throw, which is + * the bar for putting one on an error path: instrumentation must never be able + * to replace the error it is describing. */ export function deployPausedError( state: DeployOperationState, - options?: { interrupted?: boolean }, + reason: DeployPauseReason = "paused", ): DeployPausedError { - return new DeployPausedError(pausedMessage(pausedStepDescription(state)), { - exitCode: options?.interrupted ? EXIT_CODE.SIGINT : EXIT_CODE.GENERAL, - }); + if (reason !== "finalizing") setTelemetryPauseStep(state.pending.type); + return new DeployPausedError(pausedMessage(pausedStepDescription(state)), PAUSE_REASONS[reason]); } diff --git a/packages/cli-core/src/commands/doctor/README.md b/packages/cli-core/src/commands/doctor/README.md index a105a908e..2c3e1549a 100644 --- a/packages/cli-core/src/commands/doctor/README.md +++ b/packages/cli-core/src/commands/doctor/README.md @@ -98,8 +98,8 @@ clerk doctor --json --spotlight # JSON with only warnings/errors Each result includes `name`, `status` (`pass` / `warn` / `fail`), `message`, and optionally `detail` (extra diagnostic info), `remedy` -(a human-readable fix instruction), and `fix` (a label describing -the auto-fix action). +(a human-readable fix instruction), `fix` (a label describing +the auto-fix action), and `crashed` (see below). Agents cannot use `--fix` directly because the fix actions are interactive. Instead, agents should read the `remedy` field from the JSON output and @@ -108,6 +108,20 @@ or call `clerk link --app ` with a known app ID). Exit code 1 signals one or more checks failed. +## A check that crashed is not a finding + +A check that throws learned nothing about what it was meant to verify, so it is +reported as the CLI's own failure rather than as a problem with the user's +project: the result carries `crashed: true` and names the check that broke, and +the command exits with the error code `doctor_check_crashed` instead of +`doctor_failed`. It still counts as a failing check — a question was asked and +has no answer — so the exit code is unchanged. + +The name comes from the check registry in `index.ts`, because a check that +threw never returned a result to read one from. Both it and the name the check +gives its own results come from `CHECK_NAME` in `checks.ts`, so they cannot +drift apart. + ## Exit Codes | Code | Meaning | diff --git a/packages/cli-core/src/commands/doctor/check-mcp.ts b/packages/cli-core/src/commands/doctor/check-mcp.ts index 9f85f97c1..fa29b7871 100644 --- a/packages/cli-core/src/commands/doctor/check-mcp.ts +++ b/packages/cli-core/src/commands/doctor/check-mcp.ts @@ -9,6 +9,7 @@ import { collectEntries } from "../mcp/collect.ts"; import { probeMcp, type McpProbeResult } from "../mcp/probe.ts"; import type { ListEntry } from "../mcp/clients/types.ts"; +import { CHECK_NAME } from "./checks.ts"; import type { CheckResult } from "./types.ts"; type UrlProbe = { url: string; result: McpProbeResult }; @@ -60,7 +61,7 @@ export async function checkMcp(): Promise { if (failures.length > 0) { const clients = failures.map((f) => f.displayName).join(", "); return { - name: "MCP server", + name: CHECK_NAME.mcp, status: "warn", message: `Could not read the MCP config for ${clients}`, detail: [ @@ -73,7 +74,7 @@ export async function checkMcp(): Promise { if (entries.length === 0) { return { - name: "MCP server", + name: CHECK_NAME.mcp, status: "pass", message: "Skipped (no Clerk MCP entry installed)", }; @@ -81,14 +82,14 @@ export async function checkMcp(): Promise { if (unreachable.length === 0) { return { - name: "MCP server", + name: CHECK_NAME.mcp, status: "pass", message: `Reachable — ${describeReachable(probes)}`, }; } return { - name: "MCP server", + name: CHECK_NAME.mcp, status: "warn", message: describeUnreachable(unreachable, probes.length), detail: unreachable.map((p) => `${p.url}: ${describeFailure(p.result)}`).join("; "), diff --git a/packages/cli-core/src/commands/doctor/checks.ts b/packages/cli-core/src/commands/doctor/checks.ts index be54306b9..8f7e15c03 100644 --- a/packages/cli-core/src/commands/doctor/checks.ts +++ b/packages/cli-core/src/commands/doctor/checks.ts @@ -20,6 +20,28 @@ import { formatHostStateProbeFailures, getAgentHostStateProbe } from "../../lib/ import { isAgent } from "../../mode.ts"; import type { CheckResult, DoctorContext, FixAction, KeylessInstanceInfo } from "./types.ts"; +/** + * The display name of every check, in one place. + * + * A check that throws never returns a result, so `runChecks` has to name it + * from outside — and a second list of names would drift from these the first + * time one was reworded. This is that one list, read both here and by the + * registry in `index.ts`. + */ +export const CHECK_NAME = { + cliVersion: "CLI version", + hostExecution: "Host execution", + loggedIn: "Logged in", + tokenValid: "Authentication valid", + projectLinked: "Project linked", + linkedAppExists: "Application reachable", + instances: "Instance IDs", + envVars: "Environment variables", + configFile: "CLI configuration", + shellCompletion: "Shell completion", + mcp: "MCP server", +} as const; + interface CheckOptions { remedy?: string; detail?: string; @@ -92,7 +114,7 @@ async function claimHint(ctx: DoctorContext): Promise { } export async function checkLoggedIn(ctx: DoctorContext): Promise { - const check = defineCheck("Logged in", ctx.fixes.login); + const check = defineCheck(CHECK_NAME.loggedIn, ctx.fixes.login); const token = await ctx.getToken(); // Malformed-key detection is a side effect of resolving the keyless target @@ -138,7 +160,7 @@ export async function checkLoggedIn(ctx: DoctorContext): Promise { } export async function checkHostExecution(): Promise { - const check = defineCheck("Host execution"); + const check = defineCheck(CHECK_NAME.hostExecution); if (!isAgent()) { return check.pass("Skipped (human mode)"); } @@ -157,7 +179,7 @@ export async function checkHostExecution(): Promise { } export async function checkTokenValid(ctx: DoctorContext): Promise { - const check = defineCheck("Authentication valid", ctx.fixes.login); + const check = defineCheck(CHECK_NAME.tokenValid, ctx.fixes.login); const storedToken = await ctx.getToken(); if (!storedToken) { const keyless = await ctx.getKeylessTarget(); @@ -204,7 +226,7 @@ export async function checkTokenValid(ctx: DoctorContext): Promise } export async function checkProjectLinked(ctx: DoctorContext): Promise { - const check = defineCheck("Project linked", ctx.fixes.link); + const check = defineCheck(CHECK_NAME.projectLinked, ctx.fixes.link); const resolved = await ctx.getProfile(); if (resolved) { const RESOLUTION_LABELS: Record = { @@ -250,7 +272,7 @@ export async function checkProjectLinked(ctx: DoctorContext): Promise { - const check = defineCheck("Application reachable", ctx.fixes.link); + const check = defineCheck(CHECK_NAME.linkedAppExists, ctx.fixes.link); const token = await ctx.getToken(); if (!token) { // This check is account-only — the Platform API application record has no @@ -285,7 +307,7 @@ export async function checkLinkedAppExists(ctx: DoctorContext): Promise { - const check = defineCheck("Instance IDs", ctx.fixes.link); + const check = defineCheck(CHECK_NAME.instances, ctx.fixes.link); const token = await ctx.getToken(); if (!token) { // A linked profile's dev/prod instance IDs are an account-only concept — @@ -356,7 +378,7 @@ async function findEnvFile( } export async function checkEnvVars(ctx: DoctorContext): Promise { - const check = defineCheck("Environment variables", ctx.fixes.envPull); + const check = defineCheck(CHECK_NAME.envVars, ctx.fixes.envPull); const cwd = process.cwd(); const found = await findEnvFile(cwd); @@ -414,7 +436,7 @@ async function identifyEnvironment( } export async function checkConfigFile(ctx: DoctorContext): Promise { - const check = defineCheck("CLI configuration", ctx.fixes.login); + const check = defineCheck(CHECK_NAME.configFile, ctx.fixes.login); const configFile = getConfigFile(); const file = Bun.file(configFile); if (!(await file.exists())) { @@ -446,7 +468,7 @@ export async function checkConfigFile(ctx: DoctorContext): Promise // ── CLI version check ───────────────────────────────────────────────────────── export async function checkCliVersion(): Promise { - const check = defineCheck("CLI version"); + const check = defineCheck(CHECK_NAME.cliVersion); if (IS_DEV_BUILD) { return check.pass("Running development build"); } @@ -524,7 +546,7 @@ const SHELL_COMPLETION: Record< }; export async function checkShellCompletion(): Promise { - const check = defineCheck("Shell completion"); + const check = defineCheck(CHECK_NAME.shellCompletion); const shell = detectShell(); if (!shell) return check.pass("Shell completion (could not detect shell, skipped)"); diff --git a/packages/cli-core/src/commands/doctor/index.test.ts b/packages/cli-core/src/commands/doctor/index.test.ts new file mode 100644 index 000000000..65e3febd3 --- /dev/null +++ b/packages/cli-core/src/commands/doctor/index.test.ts @@ -0,0 +1,125 @@ +import { test, expect, describe, beforeEach, mock } from "bun:test"; +import { CliError, ERROR_CODE } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import type { CheckResult } from "./types.ts"; + +const actualChecks = await import("./checks.ts"); +const { CHECK_NAME } = actualChecks; + +type CheckKey = keyof typeof CHECK_NAME; +type Outcome = "pass" | "fail" | "throw"; + +/** What each check does on the next `doctor()` run; anything unset passes. */ +let outcomes: Partial> = {}; + +function stubCheck(key: CheckKey) { + return async (): Promise => { + const outcome = outcomes[key] ?? "pass"; + if (outcome === "throw") throw new Error("the check itself blew up"); + return { + name: CHECK_NAME[key], + status: outcome, + message: `${CHECK_NAME[key]}: ${outcome}`, + }; + }; +} + +// Replaced wholesale, so every export of checks.ts has to be here — the real +// module is spread back in for the non-check exports (CHECK_NAME above all, +// which check-mcp.ts also reads). +mock.module("./checks.ts", () => ({ + ...actualChecks, + checkCliVersion: stubCheck("cliVersion"), + checkHostExecution: stubCheck("hostExecution"), + checkLoggedIn: stubCheck("loggedIn"), + checkTokenValid: stubCheck("tokenValid"), + checkProjectLinked: stubCheck("projectLinked"), + checkLinkedAppExists: stubCheck("linkedAppExists"), + checkInstances: stubCheck("instances"), + checkEnvVars: stubCheck("envVars"), + checkConfigFile: stubCheck("configFile"), + checkShellCompletion: stubCheck("shellCompletion"), +})); + +mock.module("./check-mcp.ts", () => ({ checkMcp: stubCheck("mcp") })); + +const { doctor } = await import("./index.ts"); + +async function runDoctor(): Promise { + try { + await doctor(); + } catch (error) { + return error as CliError; + } + return undefined; +} + +describe("doctor", () => { + const captured = useCaptureLog(); + + beforeEach(() => { + outcomes = {}; + }); + + test("a run where every check answered and passed succeeds", async () => { + expect(await runDoctor()).toBeUndefined(); + }); + + // The two failure codes are the point: one sends the reader to their own + // project, the other to the CLI. + test("a check that found a real problem reports doctor_failed", async () => { + outcomes.envVars = "fail"; + + const error = await runDoctor(); + + expect(error?.code).toBe(ERROR_CODE.DOCTOR_FAILED); + expect(error?.message).toContain("issues with your Clerk integration"); + }); + + test("a check that threw reports doctor_check_crashed instead", async () => { + outcomes.tokenValid = "throw"; + + const error = await runDoctor(); + + expect(error?.code).toBe(ERROR_CODE.DOCTOR_CHECK_CRASHED); + }); + + // A crash outranks a finding: the run can no longer claim to have checked + // everything, so "your integration has issues" would be the wrong answer. + test("a crash alongside a real finding still reports doctor_check_crashed", async () => { + outcomes.envVars = "fail"; + outcomes.tokenValid = "throw"; + + expect((await runDoctor())?.code).toBe(ERROR_CODE.DOCTOR_CHECK_CRASHED); + }); + + test("the crashed check is named on screen, with what it threw", async () => { + outcomes.tokenValid = "throw"; + + await runDoctor(); + + expect(captured.err).toContain( + `${CHECK_NAME.tokenValid} check crashed: the check itself blew up`, + ); + expect(captured.err).not.toContain("Unknown check"); + }); + + // An agent reading `--json` gets the same distinction the exit code carries: + // `crashed` separates a broken CLI from a finding about the project, which + // the `fail` status alone cannot. + test("`--json` marks the crashed result and names it", async () => { + outcomes.mcp = "throw"; + + try { + await doctor({ json: true }); + } catch { + // the thrown failure is asserted above; this test reads the output + } + + const results = JSON.parse(captured.out) as CheckResult[]; + const crashed = results.filter((result) => result.crashed); + expect(crashed).toHaveLength(1); + expect(crashed[0]?.name).toBe(CHECK_NAME.mcp); + expect(results.every((result) => result.name !== "Unknown check")).toBe(true); + }); +}); diff --git a/packages/cli-core/src/commands/doctor/index.ts b/packages/cli-core/src/commands/doctor/index.ts index 813de75ac..a8142a974 100644 --- a/packages/cli-core/src/commands/doctor/index.ts +++ b/packages/cli-core/src/commands/doctor/index.ts @@ -6,6 +6,7 @@ import { CliError, ERROR_CODE, errorMessage } from "../../lib/errors.ts"; import { intro, outro, bar, withSpinner } from "../../lib/spinner.ts"; import { createDoctorContext } from "./context.ts"; import { + CHECK_NAME, checkLoggedIn, checkHostExecution, checkTokenValid, @@ -21,39 +22,70 @@ import { checkMcp } from "./check-mcp.ts"; import { formatCheckResult, formatJson } from "./format.ts"; import type { CheckFn, CheckResult, DoctorContext, DoctorOptions } from "./types.ts"; -const BASE_CHECKS: CheckFn[] = [ - checkCliVersion, - checkLoggedIn, - checkTokenValid, - checkProjectLinked, - checkLinkedAppExists, - checkInstances, - checkEnvVars, - checkConfigFile, - checkShellCompletion, - checkMcp, +/** + * A check paired with the name to report it under if it throws. The name it + * gives its own results comes from the same {@link CHECK_NAME} entry, so the + * two cannot disagree. + */ +type RegisteredCheck = { name: string; run: CheckFn }; + +const BASE_CHECKS: RegisteredCheck[] = [ + { name: CHECK_NAME.cliVersion, run: checkCliVersion }, + { name: CHECK_NAME.loggedIn, run: checkLoggedIn }, + { name: CHECK_NAME.tokenValid, run: checkTokenValid }, + { name: CHECK_NAME.projectLinked, run: checkProjectLinked }, + { name: CHECK_NAME.linkedAppExists, run: checkLinkedAppExists }, + { name: CHECK_NAME.instances, run: checkInstances }, + { name: CHECK_NAME.envVars, run: checkEnvVars }, + { name: CHECK_NAME.configFile, run: checkConfigFile }, + { name: CHECK_NAME.shellCompletion, run: checkShellCompletion }, + { name: CHECK_NAME.mcp, run: checkMcp }, ]; -function getChecks(): CheckFn[] { - return isAgent() ? [checkHostExecution, ...BASE_CHECKS] : BASE_CHECKS; +function getChecks(): RegisteredCheck[] { + return isAgent() + ? [{ name: CHECK_NAME.hostExecution, run: checkHostExecution }, ...BASE_CHECKS] + : BASE_CHECKS; } +/** + * A crash is a bug in the CLI, not a finding about the user's project, so it + * says which check broke instead of reporting an anonymous failure the person + * cannot act on. It still counts as a failing result: the check was asked a + * question and has no answer, and treating that as a pass would hide the one + * case where doctor itself is broken. + */ async function runChecks(ctx: DoctorContext): Promise { return Promise.all( - getChecks().map(async (check) => { + getChecks().map(async ({ name, run }) => { try { - return await check(ctx); + return await run(ctx); } catch (error) { return { - name: "Unknown check", + name, status: "fail" as const, - message: `Check crashed: ${errorMessage(error)}`, + message: `${name} check crashed: ${errorMessage(error)}`, + crashed: true as const, }; } }), ); } +/** + * What to throw for a set of results that includes a failure. A crashed check + * and a real finding are both exit 1, but they send the reader to different + * places — one is a CLI bug, the other is the user's integration — and a + * single code left them indistinguishable in telemetry and on screen. + */ +function failureCodeFor( + results: CheckResult[], +): typeof ERROR_CODE.DOCTOR_CHECK_CRASHED | typeof ERROR_CODE.DOCTOR_FAILED { + return results.some((r) => r.crashed) + ? ERROR_CODE.DOCTOR_CHECK_CRASHED + : ERROR_CODE.DOCTOR_FAILED; +} + function printResults(results: CheckResult[], options: DoctorOptions): void { for (const result of results) { if (!options.spotlight || result.status !== "pass") { @@ -127,7 +159,7 @@ export async function doctor(options: DoctorOptions = {}): Promise { const hasVerifyFailure = verifyResults.some((r) => r.status === "fail"); if (hasVerifyFailure) { throw new CliError("Some checks still failing after auto-fix", { - code: ERROR_CODE.DOCTOR_FAILED, + code: failureCodeFor(verifyResults), }); } await outro("All checks passing"); @@ -138,7 +170,7 @@ export async function doctor(options: DoctorOptions = {}): Promise { const hasFailure = allResults.some((r) => r.status === "fail"); if (hasFailure) { throw new CliError("Doctor found issues with your Clerk integration", { - code: ERROR_CODE.DOCTOR_FAILED, + code: failureCodeFor(allResults), }); } await outro("All checks passing"); diff --git a/packages/cli-core/src/commands/doctor/types.ts b/packages/cli-core/src/commands/doctor/types.ts index 1b665b8e8..99f515591 100644 --- a/packages/cli-core/src/commands/doctor/types.ts +++ b/packages/cli-core/src/commands/doctor/types.ts @@ -19,6 +19,13 @@ export interface CheckResult { detail?: string; remedy?: string; fix?: FixAction; + /** + * The check threw, so it learned nothing about what it verifies. Set + * explicitly by the only place that catches — never inferred later from the + * message text, which would make a sentence nobody knew was load-bearing + * into the contract. + */ + crashed?: true; } /** The identity of an unclaimed keyless application, fetched via its own secret key. */ diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 247e02776..7ea8e7653 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -48,6 +48,14 @@ export const ERROR_CODE = { CATALOG_ERROR: "catalog_error", /** Doctor checks found issues. */ DOCTOR_FAILED: "doctor_failed", + /** + * A doctor check threw instead of returning a result, so nothing was learned + * about what it was meant to verify. Distinct from {@link DOCTOR_FAILED}, + * which means every check ran and one of them found a real problem with the + * user's integration — a bug in the CLI and a broken project are different + * things to chase. + */ + DOCTOR_CHECK_CRASHED: "doctor_check_crashed", /** Frontend API request failed. */ FAPI_ERROR: "fapi_error", /** Subscription plan does not cover the dev instance's enabled features. */ @@ -98,6 +106,20 @@ export const ERROR_CODE = { REGISTRY_UNREACHABLE: "registry_unreachable", /** Production instance was created but came back without a domain. */ DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", + /** + * `clerk deploy` stopped with the deploy unfinished and something left for + * the user to do — a skipped OAuth provider, a DNS check they chose not to + * run. The three codes below are one situation each, rather than one code + * plus a modifier, so telling them apart never means joining `error_code` + * against `exit_code`. + */ + DEPLOY_PAUSED: "deploy_paused", + /** The user interrupted a `clerk deploy` prompt after the production instance existed. */ + DEPLOY_CANCELLED: "deploy_cancelled", + /** Every DNS component verified; Clerk had not finished provisioning yet. Nobody is waiting on the user. */ + DEPLOY_FINALIZING: "deploy_finalizing", + /** `clerk deploy` could not resolve the production instance it was about to write to. */ + DEPLOY_INSTANCE_UNRESOLVED: "deploy_instance_unresolved", /** Local publishable key and secret key address different applications. */ KEY_PAIR_MISMATCH: "key_pair_mismatch", /** BAPI returned a response the CLI could not use. */ diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 6bd6b7991..35288b2b2 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -7,6 +7,7 @@ import { declareSoftExitOutcome, finalizeAndSendTelemetry, getTelemetryStatus, + setTelemetryPauseStep, setTelemetryStage, startCommandTelemetry, telemetryEnabled, @@ -651,4 +652,27 @@ describe("finalizeAndSendTelemetry", () => { expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); }); }); + + describe("pause step", () => { + test("the step a run stopped on reaches the payload", async () => { + const payload = await sendAndCapturePayload(() => setTelemetryPauseStep("oauth"), { + outcome: "error", + exitCode: EXIT_CODE.GENERAL, + }); + expect(payload.pause_step).toBe("oauth"); + }); + + // A resume enters OAuth setup after the DNS handoff, so both steps can be + // reached in one run; the one the run actually stopped on is the last set. + test("the last step set is the one sent", async () => { + const payload = await sendAndCapturePayload( + () => { + setTelemetryPauseStep("dns"); + setTelemetryPauseStep("oauth"); + }, + { outcome: "error", exitCode: EXIT_CODE.GENERAL }, + ); + expect(payload.pause_step).toBe("oauth"); + }); + }); }); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 9c8716610..1c540c469 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -298,6 +298,20 @@ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; } +/** + * Record the step a `clerk deploy` run stopped on. Set where the pause itself + * is constructed, which is the one place that knows both that the run is + * stopping and which step it stopped on — a caller that set it earlier would + * have to unset it on every path that then carried on. + * + * Only set it for a step the *person* stopped on. A wait on Clerk's backend + * ends the run at no step at all, and leaving the last step in place there + * would count it as a drop-off nobody made. + */ +export function setTelemetryPauseStep(step: TelemetryPauseStep): void { + if (context) context.pauseStep = step; +} + /** * Declare what this run should be recorded as when it ends by setting * `process.exitCode` instead of throwing. diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts index 07abac7b5..98eaf5819 100644 --- a/packages/cli-core/src/test/integration/telemetry.test.ts +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -363,3 +363,22 @@ test("`clerk telemetry status` explains the dev-build guard", async () => { expect(result.stdout.trim()).toBe("disabled"); expect(result.stderr).toContain("dev build"); }); + +// The other half of the doctor change: a check that ran and found a real +// problem still reports `doctor_failed`. Only a crash may claim the new code, +// and nothing here crashes. +test("`doctor` with failing checks still reports doctor_failed", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.mock({ "test-telemetry.clerk.com": {} }); + + const result = await clerk.raw("doctor"); + expect(result.exitCode).toBe(1); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("doctor"); + expect(event.payload.outcome).toBe("error"); + expect(event.payload.error_code).toBe("doctor_failed"); +}); From ea5d89f955dad85861b85651d01dc2833adf4bbd Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Tue, 22 Sep 2026 23:03:31 -0700 Subject: [PATCH 04/16] Harden the telemetry test harness and fix two doc claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deploy payload tests set the capture URL without clearing `CLERK_TELEMETRY_DISABLED`, which CI sets for every job and which beats that URL, so all nine would have failed on the first pull request. The capture harness moves to `test/lib/stubs.ts`, replacing the copy in `telemetry.test.ts` as well, and clears the opt-outs itself. It classifies a normal return the way `runProgram` does rather than assuming success, requires exactly one POST carrying exactly one event, and restores `fetch`, `process.exitCode` and every env var it touched. A throw from the callback propagates unless the caller asks for it to be captured, which the old local copy did by accident and the consolidated one had stopped doing. `fakeTelemetryCommand` dropped its leftmost segment, recording `status` where a real run records `deploy status`, because telemetry excludes the root `clerk` and there was no root to discard. It synthesizes one now, and the payload-shape test pins the recorded name. `deployPausedError` becomes `throwDeployPaused(state, reason): never` with no default reason, so a pause site added later cannot compile without saying what it is, and the telemetry write cannot outlive a pause that is constructed but never thrown. Whether a reason records a step is a column in the reason table rather than a negation beside it. The doctor check list is keyed by `CHECK_NAME` and checked with `satisfies`, so a check that is named but never wired no longer compiles; `CHECK_NAME` moves to `types.ts` so `check-mcp.ts` stops importing the whole of `checks.ts` for one string. A new test pins that an agent gets the host-execution check first and a human does not get it at all — which found that the existing doctor tests had been running under whatever mode the terminal implied. Two claims were wrong: the README and the `deploy_paused` docstring said a skipped DNS check reports `deploy_paused`, when it ends the run as a success at exit 0, and a comment said the warehouse enforces which reasons carry a pause step, when it only alarms on one disappearing. Co-Authored-By: Claude Opus 5 --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/deploy/README.md | 15 +- .../src/commands/deploy/index.test.ts | 50 +----- .../cli-core/src/commands/deploy/index.ts | 25 +-- .../cli-core/src/commands/deploy/state.ts | 59 ++++--- .../cli-core/src/commands/doctor/check-mcp.ts | 3 +- .../cli-core/src/commands/doctor/checks.ts | 23 +-- .../cli-core/src/commands/doctor/context.ts | 6 +- .../src/commands/doctor/index.test.ts | 46 ++++-- .../cli-core/src/commands/doctor/index.ts | 66 +++++--- .../cli-core/src/commands/doctor/types.ts | 30 ++++ packages/cli-core/src/lib/errors.ts | 8 +- packages/cli-core/src/lib/telemetry.test.ts | 23 +-- packages/cli-core/src/test/lib/stubs.ts | 154 ++++++++++++++++-- 14 files changed, 324 insertions(+), 186 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index a4006dcf2..9e1f3bb15 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -4,4 +4,4 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Output and exit codes are unchanged. -`clerk doctor` now names the check that crashed instead of reporting "Unknown check", and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. +`clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 95efd592c..02ddc26bc 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -68,15 +68,16 @@ Telemetry's `outcome` says what happened to the _command_, not to the deploy. A `success` does not mean the deploy is finished either. `clerk deploy` under an agent prints a status report and exits 0 even when no production instance exists. How far a deploy got is carried by the `stage` and `components` payload fields, never by `outcome`. -The wizard's three non-crash endings all exit the way they always have, and are told apart by their error code rather than by `exit_code` — which cannot separate the first from the third, since both are 1. +Four ways a run ends with the deploy unfinished and nothing broken. All four exit the way they always have, and the three that exit nonzero are told apart by their error code rather than by `exit_code`, which cannot separate the first from the third since both are 1. An actual failure — an unresolvable production instance, a domain Clerk did not return — is not one of these and carries its own code. -| Ending | `error_code` | `pause_step` | Exit | -| ------------------------------------------------------------------- | ------------------- | ------------------------ | ---- | -| The user skipped an OAuth provider or a DNS check | `deploy_paused` | the step they stopped on | 1 | -| The user interrupted a prompt after the production instance existed | `deploy_cancelled` | the step they stopped on | 130 | -| Every DNS component verified, Clerk still provisioning | `deploy_finalizing` | null | 1 | +| Ending | `outcome` | `error_code` | `pause_step` | Exit | +| ------------------------------------------------------------------- | --------- | ------------------- | ------------------------ | ---- | +| The user skipped an OAuth provider | `error` | `deploy_paused` | the step they stopped on | 1 | +| The user interrupted a prompt after the production instance existed | `error` | `deploy_cancelled` | the step they stopped on | 130 | +| Every DNS component verified, Clerk still provisioning | `error` | `deploy_finalizing` | null | 1 | +| The user chose "skip" at DNS verification | `success` | null | null | 0 | -`pause_step` is null on the last row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. A Ctrl-C _before_ the production instance exists is not any of these — there is no state to preserve, so it stays a plain `abort` at exit 0. +`pause_step` is null on the finalizing row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. The DNS skip is a finished command, not a pause — the wizard prints its summary and exits 0 — so it carries no code, and it is the row to remember when the `paused` class contains no DNS traffic. A Ctrl-C _before_ the production instance exists is not any of these either — there is no state to preserve, so it stays a plain `abort` at exit 0. Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 8a0b4cc58..68c208ba8 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -2,7 +2,7 @@ import { test, expect, describe, beforeEach, afterEach, mock, spyOn } from "bun: import { mkdtemp, rm } from "node:fs/promises"; import { join, relative } from "node:path"; import { tmpdir } from "node:os"; -import { useCaptureLog, listageStubs, fakeTelemetryCommand } from "../../test/lib/stubs.ts"; +import { useCaptureLog, listageStubs, captureTelemetryPayload } from "../../test/lib/stubs.ts"; import { CliError, ERROR_CODE, EXIT_CODE, PlapiError, UserAbortError } from "../../lib/errors.ts"; const mockIsAgent = mock(); @@ -69,8 +69,6 @@ mock.module("../../lib/open.ts", () => ({ })); const { _setConfigDir, readConfig, setProfile } = await import("../../lib/config.ts"); -const { finalizeAndSendTelemetry, startCommandTelemetry, telemetryResultForError } = - await import("../../lib/telemetry.ts"); const { deploy } = await import("./index.ts"); const { providerSetupIntro, showOAuthWalkthrough } = await import("./providers.ts"); const { collectCustomDomain } = await import("./prompts.ts"); @@ -2611,48 +2609,12 @@ describe("deploy", () => { // through the telemetry context — and only the payload proves both arrive // together. describe("what telemetry records for each ending", () => { - const TELEMETRY_URL = "https://capture.invalid/v1/event"; - let realFetch: typeof globalThis.fetch; - - beforeEach(() => { - realFetch = globalThis.fetch; - }); - afterEach(() => { - globalThis.fetch = realFetch; - delete process.env.CLERK_TELEMETRY_URL; - }); - /** The event a `clerk deploy` run would post, plus whatever it threw. */ - async function deployTelemetry( - run: () => Promise, - ): Promise<{ payload: Record; error: CliError | undefined }> { - const { markTelemetryNoticeShown } = await import("../../lib/config.ts"); - await markTelemetryNoticeShown(); // past the grace run, which sends nothing - process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; - let sent: string | undefined; - globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { - sent = init.body; - return new Response("{}"); - }) as unknown as typeof fetch; - - startCommandTelemetry(fakeTelemetryCommand("deploy")); - let error: CliError | undefined; - try { - await run(); - } catch (caught) { - error = caught as CliError; - } - // `runProgram` classifies a throw and reads `process.exitCode` back - // otherwise; a wizard that returns normally never sets one. - await finalizeAndSendTelemetry( - error ? telemetryResultForError(error) : { outcome: "success", exitCode: 0 }, - ); - - expect(sent).toBeDefined(); - const parsed = JSON.parse(sent as string) as { - events: { payload: Record }[]; - }; - return { payload: parsed.events[0]!.payload, error }; + async function deployTelemetry(run: () => Promise) { + const { payload, error } = await captureTelemetryPayload("deploy", run, { + captureError: true, + }); + return { payload, error: error as CliError | undefined }; } test("a skipped OAuth provider is a paused deploy at the oauth step", async () => { diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index 8b38b1471..e8fb29505 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -58,7 +58,7 @@ import { } from "./prompts.ts"; import { DeployPausedError, - deployPausedError, + throwDeployPaused, type DeployContext, type DeployOperationState, } from "./state.ts"; @@ -439,7 +439,7 @@ async function runDnsRecordHandoff( log.blank(); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, "cancelled"); + throwDeployPaused(state, "cancelled"); } throw error; } @@ -478,7 +478,7 @@ async function runDnsVerificationPrompt( return await runDnsVerification(ctx, state); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, "cancelled"); + throwDeployPaused(state, "cancelled"); } throw error; } @@ -520,7 +520,7 @@ async function runDnsVerification( // When all DNS components are verified but the server has not yet marked the // deployment complete, the user cannot influence the remaining wait. if (outcome.status.dns && outcome.status.ssl && outcome.status.mail) { - throw deployPausedError(state, "finalizing"); + throwDeployPaused(state, "finalizing"); } if (pendingTargets.length > 0) { @@ -533,7 +533,7 @@ async function runDnsVerification( action = await chooseDnsVerificationRetryAction(); } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError(state, "cancelled"); + throwDeployPaused(state, "cancelled"); } throw error; } @@ -608,15 +608,18 @@ async function runOAuthSetup( state.frontendApiUrl, ); if (!saved) { - throw deployPausedError({ - ...state, - pending: { type: "oauth", provider: descriptor.provider }, - completedOAuthProviders: [...completed], - }); + throwDeployPaused( + { + ...state, + pending: { type: "oauth", provider: descriptor.provider }, + completedOAuthProviders: [...completed], + }, + "paused", + ); } } catch (error) { if (error instanceof UserAbortError) { - throw deployPausedError( + throwDeployPaused( { ...state, pending: { type: "oauth", provider: descriptor.provider }, diff --git a/packages/cli-core/src/commands/deploy/state.ts b/packages/cli-core/src/commands/deploy/state.ts index 1addfdcff..8a78b9282 100644 --- a/packages/cli-core/src/commands/deploy/state.ts +++ b/packages/cli-core/src/commands/deploy/state.ts @@ -55,29 +55,50 @@ export type DeployPauseReason = "paused" | "cancelled" | "finalizing"; */ const PAUSE_REASONS: Record< DeployPauseReason, - { code: ErrorCode; exitCode: typeof EXIT_CODE.GENERAL | typeof EXIT_CODE.SIGINT } + { + code: ErrorCode; + exitCode: typeof EXIT_CODE.GENERAL | typeof EXIT_CODE.SIGINT; + /** + * Whether the person stopped at a step. The CLI's own call — the warehouse + * does not enforce the pairing. Its payload contract test rejects a step + * outside `dns`/`oauth` wherever one appears, but only alarms on a step + * that stops arriving for `deploy_paused` and `deploy_cancelled` rows. So + * a fourth reason that invents a step value fails loudly, while one that + * simply needs adding to that alarm ships unmonitored until someone + * widens its eligibility list in `data-platform`. + */ + recordsPauseStep: boolean; + } > = { - paused: { code: ERROR_CODE.DEPLOY_PAUSED, exitCode: EXIT_CODE.GENERAL }, - cancelled: { code: ERROR_CODE.DEPLOY_CANCELLED, exitCode: EXIT_CODE.SIGINT }, - finalizing: { code: ERROR_CODE.DEPLOY_FINALIZING, exitCode: EXIT_CODE.GENERAL }, + paused: { code: ERROR_CODE.DEPLOY_PAUSED, exitCode: EXIT_CODE.GENERAL, recordsPauseStep: true }, + cancelled: { + code: ERROR_CODE.DEPLOY_CANCELLED, + exitCode: EXIT_CODE.SIGINT, + recordsPauseStep: true, + }, + // Every DNS component passed and the deploy is waiting on Clerk: nobody + // stopped at a step, and recording `dns` would count a drop-off nobody made. + finalizing: { + code: ERROR_CODE.DEPLOY_FINALIZING, + exitCode: EXIT_CODE.GENERAL, + recordsPauseStep: false, + }, }; /** - * The pause every unfinished `clerk deploy` run throws, and the one place that - * records which step it stopped on — the only point that knows both, since - * `state.pending` names the step and `reason` says whether the person stopped - * there at all. A `finalizing` wait did not: the deploy is waiting on Clerk, so - * recording `dns` would count a drop-off nobody made, and the code already says - * everything the step would. + * End an unfinished `clerk deploy` run, and record which step it stopped on — + * this is the only point that knows both, since `state.pending` names the step + * and `reason` says whether the person stopped there at all. * - * The telemetry write is an in-memory assignment that cannot throw, which is - * the bar for putting one on an error path: instrumentation must never be able - * to replace the error it is describing. + * Two separate guarantees keep that telemetry write safe on an error path. + * It cannot outlive a pause that isn't thrown, because this function never + * returns — the same idiom as `throwUsageError` — so no caller can construct + * the pause to inspect it and then carry on with the step left recorded. And + * it cannot replace the error it describes, because the setter is a plain + * in-memory assignment that cannot throw. */ -export function deployPausedError( - state: DeployOperationState, - reason: DeployPauseReason = "paused", -): DeployPausedError { - if (reason !== "finalizing") setTelemetryPauseStep(state.pending.type); - return new DeployPausedError(pausedMessage(pausedStepDescription(state)), PAUSE_REASONS[reason]); +export function throwDeployPaused(state: DeployOperationState, reason: DeployPauseReason): never { + const { code, exitCode, recordsPauseStep } = PAUSE_REASONS[reason]; + if (recordsPauseStep) setTelemetryPauseStep(state.pending.type); + throw new DeployPausedError(pausedMessage(pausedStepDescription(state)), { code, exitCode }); } diff --git a/packages/cli-core/src/commands/doctor/check-mcp.ts b/packages/cli-core/src/commands/doctor/check-mcp.ts index fa29b7871..c27cf8f16 100644 --- a/packages/cli-core/src/commands/doctor/check-mcp.ts +++ b/packages/cli-core/src/commands/doctor/check-mcp.ts @@ -9,8 +9,7 @@ import { collectEntries } from "../mcp/collect.ts"; import { probeMcp, type McpProbeResult } from "../mcp/probe.ts"; import type { ListEntry } from "../mcp/clients/types.ts"; -import { CHECK_NAME } from "./checks.ts"; -import type { CheckResult } from "./types.ts"; +import { CHECK_NAME, type CheckResult } from "./types.ts"; type UrlProbe = { url: string; result: McpProbeResult }; diff --git a/packages/cli-core/src/commands/doctor/checks.ts b/packages/cli-core/src/commands/doctor/checks.ts index 8f7e15c03..9b80b3617 100644 --- a/packages/cli-core/src/commands/doctor/checks.ts +++ b/packages/cli-core/src/commands/doctor/checks.ts @@ -18,30 +18,9 @@ import { } from "../../lib/update-check.ts"; import { formatHostStateProbeFailures, getAgentHostStateProbe } from "../../lib/host-execution.ts"; import { isAgent } from "../../mode.ts"; +import { CHECK_NAME } from "./types.ts"; import type { CheckResult, DoctorContext, FixAction, KeylessInstanceInfo } from "./types.ts"; -/** - * The display name of every check, in one place. - * - * A check that throws never returns a result, so `runChecks` has to name it - * from outside — and a second list of names would drift from these the first - * time one was reworded. This is that one list, read both here and by the - * registry in `index.ts`. - */ -export const CHECK_NAME = { - cliVersion: "CLI version", - hostExecution: "Host execution", - loggedIn: "Logged in", - tokenValid: "Authentication valid", - projectLinked: "Project linked", - linkedAppExists: "Application reachable", - instances: "Instance IDs", - envVars: "Environment variables", - configFile: "CLI configuration", - shellCompletion: "Shell completion", - mcp: "MCP server", -} as const; - interface CheckOptions { remedy?: string; detail?: string; diff --git a/packages/cli-core/src/commands/doctor/context.ts b/packages/cli-core/src/commands/doctor/context.ts index 9c18a30b5..60dbfef1e 100644 --- a/packages/cli-core/src/commands/doctor/context.ts +++ b/packages/cli-core/src/commands/doctor/context.ts @@ -67,9 +67,9 @@ export function createDoctorContext(): DoctorContext { // // A malformed local key (not `sk_`-prefixed) is caught here rather than // propagated: every keyless-aware check calls this getter, so letting it - // throw turns one misconfiguration into a "Check crashed" line per check, - // each stripped of its check name. It's cached as a diagnosable state - // instead, and checkLoggedIn reports it once, by name, with a remedy. + // throw turns one misconfiguration into a crashed-check line per check. + // It's cached as a diagnosable state instead, and checkLoggedIn reports + // it once, by name, with a remedy. keylessPromise = resolveKeylessTarget({ cwd: process.cwd() }).catch((error) => { if (error instanceof CliError && error.code === ERROR_CODE.INVALID_KEY_FORMAT) { keylessKeyError = error; diff --git a/packages/cli-core/src/commands/doctor/index.test.ts b/packages/cli-core/src/commands/doctor/index.test.ts index 65e3febd3..ee7da28d5 100644 --- a/packages/cli-core/src/commands/doctor/index.test.ts +++ b/packages/cli-core/src/commands/doctor/index.test.ts @@ -1,12 +1,9 @@ import { test, expect, describe, beforeEach, mock } from "bun:test"; import { CliError, ERROR_CODE } from "../../lib/errors.ts"; +import { setMode } from "../../mode.ts"; import { useCaptureLog } from "../../test/lib/stubs.ts"; -import type { CheckResult } from "./types.ts"; +import { CHECK_NAME, type CheckKey, type CheckResult } from "./types.ts"; -const actualChecks = await import("./checks.ts"); -const { CHECK_NAME } = actualChecks; - -type CheckKey = keyof typeof CHECK_NAME; type Outcome = "pass" | "fail" | "throw"; /** What each check does on the next `doctor()` run; anything unset passes. */ @@ -24,11 +21,8 @@ function stubCheck(key: CheckKey) { }; } -// Replaced wholesale, so every export of checks.ts has to be here — the real -// module is spread back in for the non-check exports (CHECK_NAME above all, -// which check-mcp.ts also reads). +// Replaced wholesale, so every export of checks.ts has to be here. mock.module("./checks.ts", () => ({ - ...actualChecks, checkCliVersion: stubCheck("cliVersion"), checkHostExecution: stubCheck("hostExecution"), checkLoggedIn: stubCheck("loggedIn"), @@ -57,8 +51,33 @@ async function runDoctor(): Promise { describe("doctor", () => { const captured = useCaptureLog(); + // Pinned rather than left to TTY detection, so the registry's agent-only + // check is included by decision, not by whether the runner has a terminal. beforeEach(() => { outcomes = {}; + setMode("human"); + }); + + async function jsonResults(): Promise { + try { + await doctor({ json: true }); + } catch { + // a failing check throws after the output is written; the output is what's read + } + return JSON.parse(captured.out) as CheckResult[]; + } + + test("an agent gets the host execution check first; a human does not get it", async () => { + setMode("agent"); + const agentNames = (await jsonResults()).map((result) => result.name); + expect(agentNames[0]).toBe(CHECK_NAME.hostExecution); + expect(agentNames).toHaveLength(Object.keys(CHECK_NAME).length); + + captured.clear(); + setMode("human"); + const humanNames = (await jsonResults()).map((result) => result.name); + expect(humanNames).not.toContain(CHECK_NAME.hostExecution); + expect(humanNames).toEqual(agentNames.slice(1)); }); test("a run where every check answered and passed succeeds", async () => { @@ -101,7 +120,6 @@ describe("doctor", () => { expect(captured.err).toContain( `${CHECK_NAME.tokenValid} check crashed: the check itself blew up`, ); - expect(captured.err).not.toContain("Unknown check"); }); // An agent reading `--json` gets the same distinction the exit code carries: @@ -110,13 +128,7 @@ describe("doctor", () => { test("`--json` marks the crashed result and names it", async () => { outcomes.mcp = "throw"; - try { - await doctor({ json: true }); - } catch { - // the thrown failure is asserted above; this test reads the output - } - - const results = JSON.parse(captured.out) as CheckResult[]; + const results = await jsonResults(); const crashed = results.filter((result) => result.crashed); expect(crashed).toHaveLength(1); expect(crashed[0]?.name).toBe(CHECK_NAME.mcp); diff --git a/packages/cli-core/src/commands/doctor/index.ts b/packages/cli-core/src/commands/doctor/index.ts index a8142a974..f1574c68d 100644 --- a/packages/cli-core/src/commands/doctor/index.ts +++ b/packages/cli-core/src/commands/doctor/index.ts @@ -6,7 +6,6 @@ import { CliError, ERROR_CODE, errorMessage } from "../../lib/errors.ts"; import { intro, outro, bar, withSpinner } from "../../lib/spinner.ts"; import { createDoctorContext } from "./context.ts"; import { - CHECK_NAME, checkLoggedIn, checkHostExecution, checkTokenValid, @@ -20,32 +19,44 @@ import { } from "./checks.ts"; import { checkMcp } from "./check-mcp.ts"; import { formatCheckResult, formatJson } from "./format.ts"; -import type { CheckFn, CheckResult, DoctorContext, DoctorOptions } from "./types.ts"; +import { + CHECK_NAME, + type CheckFn, + type CheckKey, + type CheckResult, + type DoctorContext, + type DoctorOptions, +} from "./types.ts"; + +/** + * Every check, keyed by its entry in {@link CHECK_NAME} so the compiler rejects + * a missing one — before this, a check that was exported but never listed + * simply did not run, and nothing said so. Listed in the order they run; that + * order is read from here, not from `CHECK_NAME`. `hostExecution` leads + * because it runs first under an agent and not at all for a human. + */ +const CHECKS = { + hostExecution: checkHostExecution, + cliVersion: checkCliVersion, + loggedIn: checkLoggedIn, + tokenValid: checkTokenValid, + projectLinked: checkProjectLinked, + linkedAppExists: checkLinkedAppExists, + instances: checkInstances, + envVars: checkEnvVars, + configFile: checkConfigFile, + shellCompletion: checkShellCompletion, + mcp: checkMcp, +} satisfies Record; /** - * A check paired with the name to report it under if it throws. The name it - * gives its own results comes from the same {@link CHECK_NAME} entry, so the - * two cannot disagree. + * Each check paired with the name to report it under if it throws. A check + * names its own results from the same `CHECK_NAME` entry, so the two agree. */ -type RegisteredCheck = { name: string; run: CheckFn }; - -const BASE_CHECKS: RegisteredCheck[] = [ - { name: CHECK_NAME.cliVersion, run: checkCliVersion }, - { name: CHECK_NAME.loggedIn, run: checkLoggedIn }, - { name: CHECK_NAME.tokenValid, run: checkTokenValid }, - { name: CHECK_NAME.projectLinked, run: checkProjectLinked }, - { name: CHECK_NAME.linkedAppExists, run: checkLinkedAppExists }, - { name: CHECK_NAME.instances, run: checkInstances }, - { name: CHECK_NAME.envVars, run: checkEnvVars }, - { name: CHECK_NAME.configFile, run: checkConfigFile }, - { name: CHECK_NAME.shellCompletion, run: checkShellCompletion }, - { name: CHECK_NAME.mcp, run: checkMcp }, -]; - -function getChecks(): RegisteredCheck[] { - return isAgent() - ? [{ name: CHECK_NAME.hostExecution, run: checkHostExecution }, ...BASE_CHECKS] - : BASE_CHECKS; +function getChecks(): { name: string; run: CheckFn }[] { + return (Object.keys(CHECKS) as CheckKey[]) + .filter((key) => key !== "hostExecution" || isAgent()) + .map((key) => ({ name: CHECK_NAME[key], run: CHECKS[key] })); } /** @@ -77,6 +88,13 @@ async function runChecks(ctx: DoctorContext): Promise { * and a real finding are both exit 1, but they send the reader to different * places — one is a CLI bug, the other is the user's integration — and a * single code left them indistinguishable in telemetry and on screen. + * + * Decided from one result set. After `--fix`, that is the verify pass alone: + * it re-runs every check, so it is the complete answer and the screen the + * user last saw. The cost is that a first-pass crash the verify pass does not + * reproduce is recorded nowhere — a transient one is superseded by whatever + * durable finding remained, which is why `doctor_check_crashed` rows can be + * rarer than crashes people report. */ function failureCodeFor( results: CheckResult[], diff --git a/packages/cli-core/src/commands/doctor/types.ts b/packages/cli-core/src/commands/doctor/types.ts index 99f515591..eadf7b331 100644 --- a/packages/cli-core/src/commands/doctor/types.ts +++ b/packages/cli-core/src/commands/doctor/types.ts @@ -1,8 +1,38 @@ +/** + * The doctor module's shared contract: the result and context types every + * check is written against, and — the one runtime export — the display name + * of each check, which both the checks and the registry in `index.ts` read. + */ import type { resolveProfile } from "../../lib/config.ts"; import type { CliError } from "../../lib/errors.ts"; import type { Application } from "../../lib/plapi.ts"; import type { KeylessTarget } from "../../lib/keyless-target.ts"; +/** + * The display name of every check, in one place. + * + * A check that throws never returns a result, so `runChecks` has to name it + * from outside — and a second list of names would drift from these the first + * time one was reworded. This is that one list, read by the checks and by the + * registry in `index.ts`. Declaration order here is not read; the registry + * lists the checks in the order they run. + */ +export const CHECK_NAME = { + cliVersion: "CLI version", + hostExecution: "Host execution", + loggedIn: "Logged in", + tokenValid: "Authentication valid", + projectLinked: "Project linked", + linkedAppExists: "Application reachable", + instances: "Instance IDs", + envVars: "Environment variables", + configFile: "CLI configuration", + shellCompletion: "Shell completion", + mcp: "MCP server", +} as const; + +export type CheckKey = keyof typeof CHECK_NAME; + export type CheckStatus = "pass" | "warn" | "fail"; export type ResolvedProfile = NonNullable>>; diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 7ea8e7653..6d7eb9209 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -108,10 +108,10 @@ export const ERROR_CODE = { DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", /** * `clerk deploy` stopped with the deploy unfinished and something left for - * the user to do — a skipped OAuth provider, a DNS check they chose not to - * run. The three codes below are one situation each, rather than one code - * plus a modifier, so telling them apart never means joining `error_code` - * against `exit_code`. + * the user to do: they skipped an OAuth provider. (Skipping the DNS check is + * not a pause — the run carries on and ends as a success.) The three codes + * below are one situation each, rather than one code plus a modifier, so + * telling them apart never means joining `error_code` against `exit_code`. */ DEPLOY_PAUSED: "deploy_paused", /** The user interrupted a `clerk deploy` prompt after the production instance existed. */ diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 35288b2b2..7a492627e 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -19,7 +19,7 @@ import { import { ApiError, CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "./errors.ts"; import { abortInFlight, beginInterrupt, _resetInterruptState } from "./signals.ts"; import { setLogLevel } from "./log.ts"; -import { fakeTelemetryCommand, useCaptureLog } from "../test/lib/stubs.ts"; +import { captureTelemetryPayload, fakeTelemetryCommand, useCaptureLog } from "../test/lib/stubs.ts"; // Isolate config I/O (machine uuid, notice flag) from the real user config dir. let configDir: string; @@ -184,25 +184,7 @@ describe("finalizeAndSendTelemetry", () => { run: () => void | Promise, result: TelemetryResult | (() => TelemetryResult), ): Promise> { - await markTelemetryNoticeShown(); // past the grace run — reach the send path - process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; - let sent: string | undefined; - globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { - sent = init.body; - return new Response("{}"); - }) as unknown as typeof fetch; - - startCommandTelemetry(fakeCommand()); - await run(); - // Resolved after `run` so a result derived from context (the soft-exit - // declaration) sees what the run declared. - await finalizeAndSendTelemetry(typeof result === "function" ? result() : result); - - expect(sent).toBeDefined(); - const parsed = JSON.parse(sent as string) as { - events: { payload: Record }[]; - }; - return parsed.events[0]!.payload; + return (await captureTelemetryPayload("list", run, { result })).payload; } test("no-op when telemetry is disabled (no fetch, no throw)", async () => { @@ -615,6 +597,7 @@ describe("finalizeAndSendTelemetry", () => { describe("payload shape", () => { test("carries exactly the agreed keys", async () => { const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); + expect(payload.command).toBe("list"); expect(Object.keys(payload).sort()).toEqual( [ "ai_agent", diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index 57dc7330d..cdded260f 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -2,7 +2,7 @@ import { Writable } from "node:stream"; import { afterEach, beforeEach, type spyOn } from "bun:test"; import { type CapturedLogs, setActiveCapture } from "../../lib/log.ts"; import { setUiOutput } from "../../lib/ui.ts"; -import type { TelemetryCommand } from "../../lib/telemetry.ts"; +import type { TelemetryCommand, TelemetryResult } from "../../lib/telemetry.ts"; export function capturedOutput(spy: ReturnType): string { return spy.mock.calls.map((c: unknown[]) => c[0]).join("\n"); @@ -247,18 +247,148 @@ export function stubFetch(impl: FetchImpl): void { /** * A stand-in for the Commander command telemetry reads, built from a space - * separated command path: `"deploy status"` yields a `status` command whose - * parent is `deploy`, which is what `startCommandTelemetry` walks to produce - * the payload's `command` field. No flags are reported as set. + * separated command path as it appears in the payload: `"deploy status"` + * yields a `status` command whose parent is `deploy`, whose parent is the + * root `clerk`. No flags are reported as set. + * + * The root is synthesized rather than taken from `path` because telemetry + * walks parents and stops at the one with no parent — it excludes the root + * `clerk` from what it records. Without a root to discard, the leftmost + * segment was discarded instead, so `"deploy status"` recorded `status` and + * a single-segment `"deploy"` recorded the empty string. */ export function fakeTelemetryCommand(path: string): TelemetryCommand { const noOptions = { options: [] as never[], getOptionValueSource: () => undefined }; - // Root first, so each command's parent is the segment to its left. The - // outermost parent is null: telemetry excludes the root `clerk` itself. - return path - .split(" ") - .reduce( - (parent, segment) => ({ name: () => segment, ...noOptions, parent }), - null, - ) as TelemetryCommand; + return ["clerk", ...path.split(" ")].reduce( + (parent, segment) => ({ name: () => segment, ...noOptions, parent }), + null, + ) as TelemetryCommand; +} + +/** Where the helper below points telemetry; nothing else may answer on it. */ +const TELEMETRY_CAPTURE_URL = "https://capture.invalid/v1/event"; + +type CaptureTelemetryOptions = { + /** + * Finalize with this instead of classifying what `run` did. A function is + * resolved after `run`, so one derived from context sees what it declared. + */ + result?: TelemetryResult | (() => TelemetryResult); + /** + * `run` is a command expected to report its own failure by throwing: the + * throw is caught, classified by `telemetryResultForError`, and returned. + * Without this a throw from `run` is a broken test and propagates. + */ + captureError?: boolean; +}; + +/** + * Run `run` inside a telemetry context for `command` and return the payload + * of the one event finalizing it would post. + * + * Models the two `runProgram` branches that send an event: a throw is + * classified by `telemetryResultForError` (opt in with `captureError`), a + * normal return by `telemetryResultForSoftExit` reading `process.exitCode` + * back. It does not model the third — a latched Ctrl-C, on which `runProgram` + * sends nothing — so a test of a real interrupt must not expect a payload. + * + * Exactly one POST carrying exactly one event is required, which is the + * one-terminal-event-per-run rule asserted rather than assumed. Only requests + * to the capture URL count; anything else `run` fetches is delegated to + * whatever `fetch` the caller already had installed. + * + * Self-contained on purpose. CI sets `CLERK_TELEMETRY_DISABLED` for every + * job, and that opt-out beats the capture URL, so the opt-outs are cleared + * for the duration; and several test files set `process.exitCode` without + * resetting it, so it is cleared before `run` and restored after — otherwise + * the soft-exit classification would read a leaked value. `fetch` and every + * env var touched are restored in the same `finally`. + */ +export async function captureTelemetryPayload( + command: string, + run: () => void | Promise, + options: CaptureTelemetryOptions = {}, +): Promise<{ payload: Record; error: unknown }> { + const { result, captureError = false } = options; + if (result !== undefined && captureError) { + throw new Error( + "captureTelemetryPayload: `result` overrides classification, so `captureError` would do nothing", + ); + } + + // Dynamic: a static import would load the real config module into every + // test file that imports these stubs, including the ones that mock it. + const { markTelemetryNoticeShown } = await import("../../lib/config.ts"); + const { + finalizeAndSendTelemetry, + startCommandTelemetry, + telemetryResultForError, + telemetryResultForSoftExit, + } = await import("../../lib/telemetry.ts"); + const { EXIT_CODE } = await import("../../lib/errors.ts"); + + const savedEnv = { + CLERK_TELEMETRY_URL: process.env.CLERK_TELEMETRY_URL, + CLERK_TELEMETRY_DISABLED: process.env.CLERK_TELEMETRY_DISABLED, + DO_NOT_TRACK: process.env.DO_NOT_TRACK, + }; + const savedFetch = globalThis.fetch; + const savedExitCode = process.exitCode; + const posted: string[] = []; + try { + await markTelemetryNoticeShown(); // past the grace run, which sends nothing + process.env.CLERK_TELEMETRY_URL = TELEMETRY_CAPTURE_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + process.exitCode = undefined; + globalThis.fetch = (async (url: unknown, init?: { body?: string }) => { + if (String(url) !== TELEMETRY_CAPTURE_URL) { + return savedFetch(url as Parameters[0], init as RequestInit); + } + posted.push(init?.body ?? ""); + return new Response("{}"); + }) as unknown as typeof fetch; + + startCommandTelemetry(fakeTelemetryCommand(command)); + let error: unknown; + let threw = false; + if (captureError) { + try { + await run(); + } catch (caught) { + error = caught; + threw = true; + } + } else { + // A throw here is the test itself breaking, not the command reporting a + // failure: let it out, and finalize nothing. + await run(); + } + + const resolved = typeof result === "function" ? result() : result; + await finalizeAndSendTelemetry( + resolved ?? + (threw + ? telemetryResultForError(error) + : telemetryResultForSoftExit(Number(process.exitCode ?? EXIT_CODE.SUCCESS))), + ); + + if (posted.length !== 1) { + throw new Error(`captureTelemetryPayload: expected 1 telemetry POST, got ${posted.length}`); + } + const parsed = JSON.parse(posted[0]!) as { events: { payload: Record }[] }; + if (parsed.events.length !== 1) { + throw new Error( + `captureTelemetryPayload: expected 1 event in the POST, got ${parsed.events.length}`, + ); + } + return { payload: parsed.events[0]!.payload, error }; + } finally { + globalThis.fetch = savedFetch; + process.exitCode = savedExitCode; + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } } From 167bf60591ab7ec2616e7b92590796b75c390791 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 08:57:45 -0700 Subject: [PATCH 05/16] Record the deploy state every run reaches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `clerk deploy` and `clerk deploy status` event now carries `stage`: the state the deploy itself was in when the run ended, as `clerk deploy status` would report it at that moment. A wizard run and a status check a second later agree about the same deploy, so drop-off can be counted per state for the first time. The stage is never a control-flow position. On a fresh deploy the DNS handoff runs before OAuth setup, so someone who skips a provider is at `domain_pending` with `pause_step: "oauth"` — `oauth_pending` there would contradict the status command and would credit an OAuth milestone to a deploy that never had DNS checked. Two states are established without a status read, because otherwise every run that ends before the first poll would report nothing: a fresh deploy starts at `not_started`, and the create response says whether the new instance has a domain. Everything else comes from a read that succeeded, and a run that ends before one sends null. `loadInitialDeployStatus` now reports whether its answer is live. The wizard's resume path substitutes an all-pending status when the read fails so the user can retry from the screen, and that substitute was previously indistinguishable from a genuine all-pending answer — recording it would file a network blip as a DNS stall. `recordObservedDeployStage` is the one place that check lives. Also fixes a silent test-isolation bug: Bun ignores `process.exitCode = undefined` and keeps the previous number, so the shared telemetry helper's reset never took and a run left at 1 classified later successes as errors. --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/deploy/README.md | 4 + .../src/commands/deploy/index.test.ts | 348 +++++++++++++++++- .../cli-core/src/commands/deploy/index.ts | 59 ++- .../commands/deploy/status-command.test.ts | 166 ++++++++- .../src/commands/deploy/status-command.ts | 37 +- .../src/commands/deploy/status.test.ts | 115 +++++- .../cli-core/src/commands/deploy/status.ts | 162 ++++++-- packages/cli-core/src/lib/telemetry.ts | 17 +- .../src/test/integration/telemetry.test.ts | 37 +- packages/cli-core/src/test/lib/stubs.ts | 7 +- 11 files changed, 898 insertions(+), 56 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 9e1f3bb15..51fa98062 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -2,6 +2,6 @@ "clerk": patch --- -Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Output and exit codes are unchanged. +Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Every `clerk deploy` and `clerk deploy status` event now also records the state the deploy was in when the run ended, so a run that stopped short says where. Output and exit codes are unchanged. `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 02ddc26bc..f7290ef80 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -79,6 +79,10 @@ Four ways a run ends with the deploy unfinished and nothing broken. All four exi `pause_step` is null on the finalizing row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. The DNS skip is a finished command, not a pause — the wizard prints its summary and exits 0 — so it carries no code, and it is the row to remember when the `paused` class contains no DNS traffic. A Ctrl-C _before_ the production instance exists is not any of these either — there is no state to preserve, so it stays a plain `abort` at exit 0. +`stage` is the state the deploy was in when the run ended, on every `deploy` and `deploy status` event: the same value the status report's `state` field prints, so a wizard run and a `clerk deploy status` run a second later agree about the same deploy. It is the deploy's state, not the wizard's position. On a fresh deploy the DNS handoff comes before OAuth setup, so someone who skips a provider is at `domain_pending` with `pause_step: "oauth"`; `oauth_pending` there would contradict the status command. One value per run, the last one observed. + +It is null when no reliable state was established by the time the run ended, and that null is a different answer from `not_started`. That covers a run that failed before reading anything — not linked, a failed sign-in, an API error on the first read — and two cases where a state was invalidated or never observed: a resume whose domain read failed and substituted an all-pending status so the user could retry from the screen, where the user then skipped verification; and a fresh run whose create call answered that an instance already exists, after which the resume could not read it. Two states are known without a status read: a fresh deploy starts at `not_started`, and a newly created instance is at `domain_pending` the moment Clerk returns it with a domain (`domain_provisioning` if it did not). Every other value comes from a read that succeeded. + Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: 1. `--mode` CLI flag diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 68c208ba8..ec075ff9f 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -69,7 +69,9 @@ mock.module("../../lib/open.ts", () => ({ })); const { _setConfigDir, readConfig, setProfile } = await import("../../lib/config.ts"); +const { beginInterrupt, _resetInterruptState } = await import("../../lib/signals.ts"); const { deploy } = await import("./index.ts"); +const { deployStatus } = await import("./status-command.ts"); const { providerSetupIntro, showOAuthWalkthrough } = await import("./providers.ts"); const { collectCustomDomain } = await import("./prompts.ts"); @@ -255,6 +257,7 @@ describe("deploy", () => { }); afterEach(async () => { + _resetInterruptState(); _setConfigDir(undefined); if (tempDir) { await rm(tempDir, { recursive: true, force: true }); @@ -359,9 +362,14 @@ describe("deploy", () => { } } - async function linkedProject(profile: Record = {}) { + /** A config dir with no profile for this directory. */ + async function unlinkedProject() { tempDir = await mkdtemp(join(tmpdir(), "clerk-deploy-test-")); _setConfigDir(tempDir); + } + + async function linkedProject(profile: Record = {}) { + await unlinkedProject(); const nextProfile = { workspaceId: "workspace_123", appId: "app_xyz789", @@ -2617,6 +2625,10 @@ describe("deploy", () => { return { payload, error: error as CliError | undefined }; } + // `runDnsHandoff` leaves a production instance behind, so these two are + // resumes: the live read finds the domain verified and OAuth pending, + // which is `oauth_pending` — the stage is the deploy's state, and the + // pause step is where the user was. test("a skipped OAuth provider is a paused deploy at the oauth step", async () => { await linkedProject(); mockIsAgent.mockReturnValue(false); @@ -2629,6 +2641,7 @@ describe("deploy", () => { expect(payload.outcome).toBe("error"); expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_PAUSED); expect(payload.pause_step).toBe("oauth"); + expect(payload.stage).toBe("oauth_pending"); expect(payload.exit_code).toBe(EXIT_CODE.GENERAL); }); @@ -2642,9 +2655,39 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("oauth"); + expect(payload.stage).toBe("oauth_pending"); expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); }); + // On a fresh deploy the DNS handoff comes before OAuth setup, so someone + // who skips a provider has never had DNS checked: the deploy is at + // `domain_pending`, whatever the wizard was doing. A control-flow value + // of `oauth_pending` here would contradict what `clerk deploy status` + // says about the same deploy a second later, and that agreement is the + // invariant. + test("a fresh run that skips OAuth is at domain_pending, and `deploy status` agrees", async () => { + await linkedProject(); + mockHumanFlow(); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_PAUSED); + expect(payload.pause_step).toBe("oauth"); + expect(payload.stage).toBe("domain_pending"); + + // The same deploy, read back: instance present, DNS unverified, OAuth + // unconfigured. + mockLiveProduction(); + mockGetApplicationDomainStatus.mockResolvedValue( + domainStatus({ status: "incomplete", dns: false, ssl: false, mail: false }), + ); + mockIsAgent.mockReturnValue(true); + const status = await captureTelemetryPayload("deploy status", () => deployStatus()); + + expect(status.payload.stage).toBe("domain_pending"); + expect(status.payload.outcome).toBe("incomplete"); + }); + test("Ctrl-C at the DNS retry prompt is a cancelled deploy at the dns step", async () => { await linkedProject({ instances: { development: "ins_dev_123", production: "ins_prod_123" }, @@ -2669,6 +2712,7 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("dns"); + expect(payload.stage).toBe("domain_pending"); expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); }); @@ -2685,6 +2729,7 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("dns"); + expect(payload.stage).toBe("domain_pending"); expect(payload.exit_code).toBe(EXIT_CODE.SIGINT); }); @@ -2711,6 +2756,9 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_FINALIZING); expect(payload.pause_step).toBeNull(); + // Every component passed, but the domain-status verdict is what + // counts, and Clerk has not given it. + expect(payload.stage).toBe("domain_pending"); expect(payload.exit_code).toBe(EXIT_CODE.GENERAL); }); @@ -2736,6 +2784,7 @@ describe("deploy", () => { expect(payload.exit_code).toBe(0); expect(payload.error_code).toBeNull(); expect(payload.pause_step).toBeNull(); + expect(payload.stage).toBe("domain_pending"); }); // A pause is not the only way this path fails, and the pause codes must @@ -2760,6 +2809,9 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_DOMAIN_MISSING); expect(payload.pause_step).toBeNull(); + // The instance exists and its domain does not: what `deploy status` + // reports for that, not the `domain_pending` the wizard was heading for. + expect(payload.stage).toBe("domain_provisioning"); }); // Both sites fire when the instance the wizard is about to write to @@ -2777,6 +2829,7 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED); expect(payload.exit_code).toBe(EXIT_CODE.USAGE); + expect(payload.stage).toBe("domain_pending"); }); test("an unnameable production instance at the next steps is deploy_instance_unresolved", async () => { @@ -2795,6 +2848,299 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_INSTANCE_UNRESOLVED); expect(payload.exit_code).toBe(EXIT_CODE.USAGE); + expect(payload.stage).toBe("domain_pending"); + }); + }); + + // Where each run left the deploy. `stage` is the state `clerk deploy + // status` would report at that moment, never the wizard's own position, + // and null when the run ended before any state was established — which is + // a different answer from `not_started`. + describe("what telemetry records as the stage", () => { + const noComponents = { dns: null, ssl: null, mail: null, oauth: null }; + + async function deployTelemetry(run: () => Promise) { + return captureTelemetryPayload("deploy", run, { captureError: true }); + } + + // The three endings before an instance exists. Nothing was read, but + // the wizard knows there is no production instance, which is exactly + // the condition `deploy status` reports as `not_started`. Without this + // the largest interrupt cohort would report nothing at all. + test("declining the plan is a success at not_started", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(false); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(EXIT_CODE.SUCCESS); + expect(payload.stage).toBe("not_started"); + expect(payload.components).toEqual(noComponents); + }); + + test("declining instance creation is a success at not_started", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(false); + mockInput.mockResolvedValueOnce("example.com"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("not_started"); + expect(payload.components).toEqual(noComponents); + }); + + test("Ctrl-C before the instance exists is an abort at not_started", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockRejectedValueOnce(promptExitError()); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("abort"); + expect(payload.exit_code).toBe(EXIT_CODE.SUCCESS); + expect(payload.error_code).toBeNull(); + expect(payload.stage).toBe("not_started"); + expect(payload.components).toEqual(noComponents); + }); + + // The stage comes from the create response, before the instance id is + // written to the local config: Clerk returned a domain, so the deploy is + // at `domain_pending`, and a failed local write must not file it under + // provisioning. + test("a failed local write after creation records the state the create response established", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + + const { payload, error } = await deployTelemetry(async () => { + // The next write is `persistProductionInstance` saving the profile. + writeSpy.mockImplementationOnce(() => Promise.reject(new Error("EACCES: read-only"))); + await runDeploy({}); + }); + + expect((error as Error).message).toContain("EACCES"); + expect(payload.outcome).toBe("error"); + expect(payload.stage).toBe("domain_pending"); + }); + + /** + * The create call answers that a production instance already exists, so + * the wizard resumes it. The first application read (during context + * resolution) saw no production instance; every later read sees one. + */ + function mockCreateConflict() { + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockCreateProductionInstance.mockReset(); + mockCreateProductionInstance.mockRejectedValueOnce( + new PlapiError( + 409, + JSON.stringify({ errors: [{ code: "production_instance_exists", message: "exists" }] }), + ), + ); + mockFetchApplication.mockResolvedValueOnce({ + application_id: "app_xyz789", + name: "my-saas-app", + instances: [ + { + instance_id: "ins_dev_123", + environment_type: "development", + publishable_key: "pk_test_123", + }, + ], + }); + mockLiveProduction({ + instanceId: "ins_prod_recovered", + productionConfig: { + connection_oauth_google: { + enabled: true, + client_id: "google-client-id.apps.googleusercontent.com", + client_secret: "REDACTED", + }, + }, + }); + } + + // "Instance already exists" disproves the `not_started` recorded on + // entry without saying anything about that instance's domain. If the + // resume then cannot observe a state, the run sends null — the same + // answer an ordinary resume gives — not the stale `not_started`. + test("a create conflict whose resume cannot read the domain records no stage", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockCreateConflict(); + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + mockSelect.mockResolvedValueOnce("skip"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBeNull(); + }); + + test("a create conflict whose resume reads the deploy records what it observed", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockCreateConflict(); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("complete"); + }); + + test("completing the last OAuth provider on a verified domain ends at complete", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockLiveProduction({ instanceId: "ins_prod_123" }); // Google enabled, no credentials + mockOAuthCompletion(); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("complete"); + }); + + // The resume path substitutes "everything pending" when the domain-status + // read fails, so the user can retry from the screen. That is not an + // observation: recording `domain_pending` from it would file a network + // blip as a DNS stall. + test("a resume whose domain read failed records no stage when DNS is then skipped", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + mockConfirm.mockResolvedValueOnce(false); + mockSelect.mockResolvedValueOnce("skip"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBeNull(); + }); + + test("a resume whose domain read failed records what a later successful poll observes", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockGetApplicationDomainStatus + .mockRejectedValueOnce( + new PlapiError( + 500, + JSON.stringify({ errors: [{ code: "server_error" }] }), + "https://x", + ), + ) + .mockResolvedValue( + domainStatus({ status: "complete", dns: true, ssl: true, mail: true }), + ); + mockConfirm.mockResolvedValueOnce(false); + mockSelect.mockResolvedValueOnce("check"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("complete"); + }); + + // The stage is recorded per poll, not once the wait returns: the signal + // handler reports a Ctrl-C with the stage as it stands at that moment. + // Starting from a failed resume read, so the only way `domain_pending` + // can get there is from the poll that completed before the interrupt. + test("Ctrl-C mid-poll records what the last completed poll observed", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + let reads = 0; + mockGetApplicationDomainStatus.mockImplementation(() => { + reads++; + if (reads === 1) { + throw new PlapiError( + 500, + JSON.stringify({ errors: [{ code: "server_error" }] }), + "https://x", + ); + } + if (reads === 2) { + return domainStatus({ status: "incomplete", dns: false, ssl: false, mail: false }); + } + beginInterrupt(); + throw new DOMException("The operation was aborted.", "AbortError"); + }); + mockConfirm.mockResolvedValueOnce(false); + mockSelect.mockResolvedValueOnce("check"); + + const { payload, error } = await deployTelemetry(async () => runDeploy({})); + + expect(error).toBeInstanceOf(DOMException); + expect(payload.stage).toBe("domain_pending"); + }); + + test("not linked fails before any state and records null", async () => { + await unlinkedProject(); + mockIsAgent.mockReturnValue(false); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.NOT_LINKED); + expect(payload.stage).toBeNull(); + }); + + test("a failure before the wizard starts records null", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockFetchApplication.mockRejectedValue( + new PlapiError( + 401, + JSON.stringify({ errors: [{ code: "authentication_invalid" }] }), + "https://x", + ), + ); + + const { payload, error } = await deployTelemetry(async () => runDeploy({})); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.stage).toBeNull(); + }); + + // The agent handoff is a successful command whatever the deploy's state, + // so the stage is the only thing that says how far the deploy got. + test("agent mode with no production instance is a success at not_started", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(true); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(EXIT_CODE.SUCCESS); + expect(payload.stage).toBe("not_started"); + expect(payload.components).toEqual(noComponents); + }); + + test("agent mode on a finished deploy is a success at complete", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_mock" }, + }); + mockIsAgent.mockReturnValue(true); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("complete"); }); }); }); diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index e8fb29505..e1315bed4 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -65,14 +65,20 @@ import { import { buildDeployStatusReport, loadDevelopmentOAuthProviders, + recordDeployStage, + recordObservedDeployStage, + resolveActiveReportState, resolveDeployContext, resolveDeployState, resolveLiveApplicationContext, resolveLiveDeploySnapshot, + retractDeployStage, waitForDeployStatus, + type DeployProgressHandlers, type DeployStatusOutcome, type DiscoveredOAuthProviders, type LiveDeploySnapshot, + type OAuthSetupFacts, } from "./status.ts"; type DeployOptions = Record; @@ -124,6 +130,7 @@ async function emitAgentDeployHandoff(): Promise { } const state = await resolveDeployState(ctx); + recordObservedDeployStage(state, null); const report = buildDeployStatusReport(state, null); log.data(JSON.stringify(report, null, 2)); } @@ -145,6 +152,11 @@ async function runDeploy(ctx: DeployContext): Promise { } async function startNewDeploy(ctx: DeployContext): Promise { + // What `clerk deploy status` reports on exactly this condition. Set before + // anything is read, so a run that ends at the plan, the domain prompt or the + // create confirmation says where the deploy stood rather than nothing. + recordDeployStage("not_started"); + const { descriptors: oauthProviders, unsupported }: DiscoveredOAuthProviders = await loadDevelopmentOAuthProviders(ctx); @@ -171,6 +183,10 @@ async function startNewDeploy(ctx: DeployContext): Promise { const productionOrExists = await createProductionInstance(ctx, domain); if (productionOrExists === "exists") { + // `not_started` is now disproven, and nothing replaces it until the resume + // below reads the instance. If that read fails, or substitutes, the run + // ends with no stage rather than a false one. + retractDeployStage(); log.blank(); log.info( "A production instance already exists for this application. Resuming the existing deploy.", @@ -187,6 +203,14 @@ async function startNewDeploy(ctx: DeployContext): Promise { return; } const production = productionOrExists; + // From the create response, before anything local happens: the instance + // exists, and the response says whether it has a domain. `deploy status` + // reads both from the API, so it agrees from this instant on, and a failure + // persisting the id below records the deploy's state rather than filing a + // local disk write under provisioning. Without this, every run that ends + // before the first DNS poll — which comes after OAuth setup — would report + // no stage, and that is where the wizard loses people. + recordDeployStage(production.active_domain ? "domain_pending" : "domain_provisioning"); await persistProductionInstance(ctx, production.id); // "Clerk production instance", not just "production instance": the user // also has a deployment on their host, and this is the one Clerk manages. @@ -243,7 +267,12 @@ async function startNewDeploy(ctx: DeployContext): Promise { completedOAuthProviders, }); - await finishDeploy(ctx, productionDomain, completedOAuthProviders, dnsStatus); + await finishDeploy( + ctx, + productionDomain, + { oauthProviders: operationState.oauthProviders, completedOAuthProviders }, + dnsStatus, + ); } async function reconcileExistingDeploy(ctx: DeployContext): Promise { @@ -253,12 +282,16 @@ async function reconcileExistingDeploy(ctx: DeployContext): Promise { const snapshot = await resolveLiveDeploySnapshot(ctx); if (!snapshot) { + recordDeployStage("domain_provisioning"); log.blank(); log.info("A production instance exists, but Clerk did not return a production domain yet."); log.info("Run `clerk deploy` again after the domain is available from the API."); await outro("No deploy actions available"); return; } + // Records nothing when the domain read was substituted; a later poll that + // succeeds will. + recordObservedDeployStage({ kind: "active", snapshot }, null); log.blank(); for (const line of printPlan(ctx.appLabel, buildLiveDeployPlan(snapshot))) { @@ -270,7 +303,7 @@ async function reconcileExistingDeploy(ctx: DeployContext): Promise { if (!snapshot.pending) { log.info("No deploy actions remain."); - await finishDeploy(ctx, snapshot.domain, snapshot.completedOAuthProviders, "verified"); + await finishDeploy(ctx, snapshot.domain, snapshot, "verified"); return; } @@ -308,7 +341,7 @@ async function reconcileExistingDeploy(ctx: DeployContext): Promise { ); } - await finishDeploy(ctx, snapshot.domain, snapshot.completedOAuthProviders, dnsStatus); + await finishDeploy(ctx, snapshot.domain, snapshot, dnsStatus); } type DnsVerificationResult = "verified" | "pending"; @@ -491,7 +524,11 @@ async function runDnsVerification( const domainIdOrName = state.productionDomainId ?? state.domain; while (true) { - const outcome = await pollDeployStatus(ctx.appId, domainIdOrName, state.domain); + // Per poll, not once the wait returns: a Ctrl-C mid-wait is reported by + // the signal handler with the stage as it stands at that moment. + const outcome = await pollDeployStatus(ctx.appId, domainIdOrName, state.domain, (polled) => + recordDeployStage(resolveActiveReportState(state, polled.verified)), + ); if (outcome.verified) { log.blank(); @@ -549,10 +586,12 @@ async function pollDeployStatus( appId: string, domainIdOrName: string, domain: string, + onStatus: DeployProgressHandlers["onStatus"], ): Promise { return waitForDeployStatus(appId, domainIdOrName, domain, { runVerification: async (progressLabel, work) => withSpinner(progressLabel, work), onVerified: () => log.success(deployComponentLabels("dns", domain).done), + onStatus, }); } @@ -695,13 +734,21 @@ async function persistProductionInstance(ctx: DeployContext, productionInstanceI async function finishDeploy( ctx: DeployContext, domain: string, - completedOAuthProviders: readonly string[], + oauth: OAuthSetupFacts, dnsStatus: DnsVerificationResult, ): Promise { + // A verified domain is an observation — a poll's, or a live resume read's — + // so the resolver decides between `complete` and `oauth_pending` from the + // facts rather than this function assuming OAuth finished. (Today it always + // has: `runOAuthSetup` pauses rather than return a partial set.) A pending + // domain adds nothing: the last set point already recorded it, or, after a + // substituted resume read, deliberately left it unrecorded. + if (dnsStatus === "verified") recordDeployStage(resolveActiveReportState(oauth, true)); + log.blank(); for (const line of productionSummary( domain, - completedOAuthProviders.map((provider) => providerLabel(provider)), + oauth.completedOAuthProviders.map((provider) => providerLabel(provider)), dnsStatus, )) { log.info(line); diff --git a/packages/cli-core/src/commands/deploy/status-command.test.ts b/packages/cli-core/src/commands/deploy/status-command.test.ts index 9a2036f6b..97b01f3a0 100644 --- a/packages/cli-core/src/commands/deploy/status-command.test.ts +++ b/packages/cli-core/src/commands/deploy/status-command.test.ts @@ -3,7 +3,12 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { EXIT_CODE, PlapiError } from "../../lib/errors.ts"; -import { fakeTelemetryCommand, stubFetch, useCaptureLog } from "../../test/lib/stubs.ts"; +import { + captureTelemetryPayload, + fakeTelemetryCommand, + stubFetch, + useCaptureLog, +} from "../../test/lib/stubs.ts"; const mockFetchApplication = mock(); const mockListApplicationDomains = mock(); @@ -22,7 +27,8 @@ mock.module("../../lib/sleep.ts", () => ({ const { _setConfigDir, setProfile } = await import("../../lib/config.ts"); const { setMode } = await import("../../mode.ts"); -const { beginInterrupt, _resetInterruptState } = await import("../../lib/signals.ts"); +const { beginInterrupt, interruptedExitCode, _resetInterruptState } = + await import("../../lib/signals.ts"); const { deployStatus, humanNextAction } = await import("./status-command.ts"); const { startCommandTelemetry, telemetryResultForSoftExit } = await import("../../lib/telemetry.ts"); @@ -125,7 +131,8 @@ describe("deploy status", () => { captured.clear(); setMode("agent"); exitCodeBefore = process.exitCode; - process.exitCode = undefined; + // Bun ignores `process.exitCode = undefined`; only a number resets it. + process.exitCode = EXIT_CODE.SUCCESS; process.env.CLERK_PLATFORM_API_KEY = "ak_test"; stubFetch((...args) => routePlapiFetch(...args)); tempDir = await mkdtemp(join(tmpdir(), "clerk-status-test-")); @@ -464,6 +471,30 @@ describe("deploy status", () => { expect(payload.domainStatus).toEqual({ dns: "complete", ssl: "pending", mail: "complete" }); }); + // The interrupted report carries the last observation rather than a + // hardcoded "not verified", so a deploy that was already complete when the + // interrupt landed says so — the same thing telemetry records for it. The + // exit code is what tells a script the command did not finish. + test("agent mode Ctrl-C mid-wait on an already complete deploy still reports complete", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + let reads = 0; + mockGetApplicationDomainStatus.mockImplementation(() => { + reads++; + if (reads === 1) return completeDomainStatus(); // the state read + beginInterrupt(); + throw abortError(); + }); + + await expect(deployStatus({ wait: true })).rejects.toThrow(); + + const payload = JSON.parse(captured.out); + expect(payload).toMatchObject({ complete: true, state: "complete" }); + expect(interruptedExitCode()).toBe(EXIT_CODE.SIGINT); + }); + test("human mode Ctrl-C during the preflight prints only the next action", async () => { setMode("human"); mockFetchApplication.mockResolvedValue(appWith(true)); @@ -792,6 +823,135 @@ describe("deploy status", () => { exitCode: EXIT_CODE.GENERAL, }); }); + + // `stage` is the state the deploy was in when the run ended — the same + // value the report's `state` field prints — and null when the run failed + // before it had one. Read off the posted payload, since the stage travels + // through the telemetry context rather than the report. + describe("stage", () => { + function statusTelemetry(options: Parameters[0] = {}) { + return captureTelemetryPayload("deploy status", () => deployStatus(options), { + captureError: true, + }); + } + + test("no production instance is not_started", async () => { + mockFetchApplication.mockResolvedValue(appWith(false)); + + const { payload } = await statusTelemetry(); + + expect(payload.stage).toBe("not_started"); + expect(payload.outcome).toBe("incomplete"); + }); + + test("an instance without a domain yet is domain_provisioning", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockListApplicationDomains.mockResolvedValue({ data: [], total_count: 0 }); + + const { payload } = await statusTelemetry(); + + expect(payload.stage).toBe("domain_provisioning"); + }); + + test("unverified DNS is domain_pending", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(pendingDnsDomainStatus()); + + const { payload } = await statusTelemetry(); + + expect(payload.stage).toBe("domain_pending"); + }); + + test("a verified domain still missing OAuth credentials is oauth_pending", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockFetchInstanceConfig.mockImplementation(() => ({ + connection_oauth_google: { enabled: true }, + })); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(completeDomainStatus()); + + const { payload } = await statusTelemetry(); + + expect(payload.stage).toBe("oauth_pending"); + }); + + test("a finished deploy is complete", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(completeDomainStatus()); + + const { payload } = await statusTelemetry(); + + expect(payload.stage).toBe("complete"); + expect(payload.outcome).toBe("success"); + }); + + test("under --wait the last poll's state is recorded", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + mockGetApplicationDomainStatus + .mockResolvedValueOnce(pendingDnsDomainStatus()) + .mockResolvedValueOnce(pendingSslDomainStatus()) + .mockResolvedValue(completeDomainStatus()); + + const { payload } = await statusTelemetry({ wait: true }); + + expect(payload.stage).toBe("complete"); + expect(payload.outcome).toBe("success"); + }); + + test("Ctrl-C mid-wait keeps the state the last completed poll established", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + let polls = 0; + mockGetApplicationDomainStatus.mockImplementation(() => { + polls++; + if (polls <= 2) return pendingDnsDomainStatus(); + beginInterrupt(); + throw abortError(); + }); + + const { payload, error } = await statusTelemetry({ wait: true }); + + expect(error).toBeInstanceOf(DOMException); + expect(payload.stage).toBe("domain_pending"); + }); + + test("not linked fails before any state and records null", async () => { + _setConfigDir(tempDir); + await rm(join(tempDir, "config.json"), { force: true }); + + const { payload } = await statusTelemetry(); + + expect(payload.error_code).toBe("not_linked"); + expect(payload.stage).toBeNull(); + }); + + test("a failed state read records null, not the state it was about to read", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + + const { payload, error } = await statusTelemetry(); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.stage).toBeNull(); + }); + }); }); }); diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index 3a1438911..a9a29c1ac 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -5,12 +5,13 @@ import { interruptedExitCode } from "../../lib/signals.ts"; import { sleep } from "../../lib/sleep.ts"; import { withSpinner } from "../../lib/spinner.ts"; import { declareSoftExitOutcome } from "../../lib/telemetry.ts"; -import { deployComponentLabels, dnsRecords, type DeployComponentStatus } from "./copy.ts"; +import { deployComponentLabels, dnsRecords } from "./copy.ts"; import { buildDeployStatusReport, buildInterruptedDeployStatusReport, deployNextStep, loadProductionDomain, + recordObservedDeployStage, resolveDeployContext, resolveDeployState, triggerDeployStatusCheck, @@ -35,12 +36,12 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { - lastPolledStatus = status; + onStatus: (polled) => { + lastPolled = polled; + recordObservedDeployStage(active, polled); }, }); } @@ -80,9 +87,12 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { @@ -117,7 +126,7 @@ async function runPreflightDeployStatusCheck(ctx: DeployContext): Promise, - options: { triggerCheck?: boolean; onStatus?: (status: DeployComponentStatus) => void } = {}, + options: { triggerCheck?: boolean; onStatus?: (outcome: DeployStatusOutcome) => void } = {}, ): Promise { const { snapshot } = state; const domainIdOrName = snapshot.productionDomainId ?? snapshot.domain; diff --git a/packages/cli-core/src/commands/deploy/status.test.ts b/packages/cli-core/src/commands/deploy/status.test.ts index 4325335cf..7be255bb8 100644 --- a/packages/cli-core/src/commands/deploy/status.test.ts +++ b/packages/cli-core/src/commands/deploy/status.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { PlapiError } from "../../lib/errors.ts"; +import { fakeTelemetryCommand } from "../../test/lib/stubs.ts"; import type { LiveDeploySnapshot } from "./status.ts"; const mockFetchApplication = mock(); @@ -24,9 +25,13 @@ const { buildDeployStatusReport, buildInterruptedDeployStatusReport, deployNextStep, + loadInitialDeployStatus, + recordObservedDeployStage, resolveDeployState, waitForDeployStatus, } = await import("./status.ts"); +const { currentTelemetryStage, setTelemetryStage, startCommandTelemetry } = + await import("../../lib/telemetry.ts"); const ctx = { profileKey: "/tmp/x", @@ -166,7 +171,11 @@ describe("waitForDeployStatus", () => { mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeStatus); mockGetApplicationDomainStatus.mockResolvedValue(completeStatus); - const outcome = await waitForDeployStatus("app_1", "dmn_1", "example.com", passthroughHandlers); + const observed: unknown[] = []; + const outcome = await waitForDeployStatus("app_1", "dmn_1", "example.com", { + ...passthroughHandlers, + onStatus: (polled) => observed.push(polled), + }); expect(mockTriggerApplicationDomainDNSCheck).toHaveBeenCalledWith("app_1", "dmn_1"); expect(mockTriggerApplicationDomainDNSCheck.mock.invocationCallOrder[0]).toBeLessThan( @@ -176,6 +185,9 @@ describe("waitForDeployStatus", () => { verified: true, status: { dns: true, ssl: true, mail: true }, }); + // The poll's own verdict travels with its components, so an observer can + // tell "verified" from "all three passed but Clerk is still finalizing". + expect(observed).toEqual([outcome]); }); test("continues polling when the DNS check is already in flight", async () => { @@ -193,6 +205,105 @@ describe("waitForDeployStatus", () => { }); }); +// The wizard's resume path substitutes "everything pending" for a failed read +// so the user can retry from the screen. The substitute is indistinguishable +// from a real all-pending answer by its fields, so the flag is the only thing +// that stops it being recorded as an observation. +describe("loadInitialDeployStatus", () => { + test("a successful read is live", async () => { + mockGetApplicationDomainStatus.mockResolvedValue(completeStatus); + + const result = await loadInitialDeployStatus("app_1", "dmn_1"); + + expect(result.live).toBe(true); + expect(result.status).toEqual(completeStatus as typeof result.status); + }); + + test("a failed read is substituted with everything pending, and is not live", async () => { + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + + const result = await loadInitialDeployStatus("app_1", "dmn_1"); + + expect(result.live).toBe(false); + expect(result.status.status).toBe("incomplete"); + expect(result.status.dns?.status).toBe("not_started"); + }); + + test("the read-only path throws instead of substituting", async () => { + mockGetApplicationDomainStatus.mockRejectedValue( + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"), + ); + + await expect( + loadInitialDeployStatus("app_1", "dmn_1", { throwOnStatusError: true }), + ).rejects.toBeInstanceOf(PlapiError); + }); +}); + +// The only guard between a substituted "everything pending" read and a +// recorded DNS stall. The status command never trips it today because its +// read throws instead of substituting; the guard is here so that a change to +// that read cannot silently start recording fallbacks. +describe("recordObservedDeployStage", () => { + const snapshot = { + appId: "app_1", + developmentInstanceId: "ins_dev", + productionInstanceId: "ins_prod", + productionDomainId: "dmn_1", + domain: "example.com", + oauthProviders: ["google"], + oauthProviderDescriptors: [], + completedOAuthProviders: ["google"], + cnameTargets: [], + domainComplete: false, + live: true, + componentStatus: { dns: false, ssl: false, mail: false }, + unsupportedOAuthProviderCount: 0, + unsupportedOAuthProviders: [], + pending: { type: "dns" as const }, + } satisfies LiveDeploySnapshot; + + beforeEach(() => { + startCommandTelemetry(fakeTelemetryCommand("deploy status")); + }); + + test("a live snapshot records its state", () => { + recordObservedDeployStage({ kind: "active", snapshot }, null); + + expect(currentTelemetryStage()).toBe("domain_pending"); + }); + + test("a substituted snapshot leaves the stage as it was", () => { + setTelemetryStage("oauth_pending"); + + recordObservedDeployStage({ kind: "active", snapshot: { ...snapshot, live: false } }, null); + + expect(currentTelemetryStage()).toBe("oauth_pending"); + }); + + test("a poll outcome is its own observation, whatever the snapshot was", () => { + recordObservedDeployStage( + { kind: "active", snapshot: { ...snapshot, live: false } }, + { verified: true, status: { dns: true, ssl: true, mail: true } }, + ); + + expect(currentTelemetryStage()).toBe("complete"); + }); + + test("the two states without a snapshot record directly", () => { + recordObservedDeployStage({ kind: "not_started" }, null); + expect(currentTelemetryStage()).toBe("not_started"); + + recordObservedDeployStage( + { kind: "domain_provisioning", appId: "app_1", productionInstanceId: "ins_prod" }, + null, + ); + expect(currentTelemetryStage()).toBe("domain_provisioning"); + }); +}); + describe("buildDeployStatusReport", () => { const activeSnapshot = { appId: "app_1", @@ -208,6 +319,7 @@ describe("buildDeployStatusReport", () => { { host: "clkmail.example.com", value: "mail.clerk.services", required: true }, ], domainComplete: false, + live: true, componentStatus: { dns: false, ssl: false, mail: false }, unsupportedOAuthProviderCount: 0, unsupportedOAuthProviders: [], @@ -510,6 +622,7 @@ describe("report urls", () => { completedOAuthProviders: [], cnameTargets: [], domainComplete: false, + live: true, componentStatus: { dns: false, ssl: false, mail: false }, unsupportedOAuthProviderCount: 0, unsupportedOAuthProviders: [], diff --git a/packages/cli-core/src/commands/deploy/status.ts b/packages/cli-core/src/commands/deploy/status.ts index 341a9f430..2d9bd6f02 100644 --- a/packages/cli-core/src/commands/deploy/status.ts +++ b/packages/cli-core/src/commands/deploy/status.ts @@ -33,6 +33,7 @@ import { type OAuthProviderDescriptor, } from "./providers.ts"; import type { DeployContext, DeployOperationState } from "./state.ts"; +import { clearTelemetryStage, setTelemetryStage } from "../../lib/telemetry.ts"; const DEPLOY_STATUS_INITIAL_RETRY_DELAY_MS = 3000; const DEPLOY_STATUS_MAX_RETRIES = 5; @@ -48,8 +49,10 @@ export interface DeployProgressHandlers { * Fires every time a poll resolves a fresh status. Ctrl-C rejects out of the * next poll or its countdown, discarding the loop's local status, so a caller * that wants to report partial progress on interrupt has to capture it here. + * Carries the poll's verdict on the domain as well as its components: all + * three can be verified while Clerk is still finalizing. */ - onStatus?(status: DeployComponentStatus): void; + onStatus?(outcome: DeployStatusOutcome): void; } export type DeployStatusOutcome = { verified: boolean; status: DeployComponentStatus }; @@ -133,6 +136,15 @@ export type LiveDeploySnapshot = Omit< completedOAuthProviders: OAuthProvider[]; domainComplete: boolean; componentStatus: DeployComponentStatus; + /** + * Whether `domainComplete` and `componentStatus` come from a domain-status + * read that succeeded. The wizard's resume path substitutes "everything + * pending" when that read fails so the user can retry from the screen, and + * the substitute is byte-identical to a genuine all-pending answer; this is + * the only thing that tells them apart. Nothing about the domain may be + * recorded from a snapshot that is not live. + */ + live: boolean; unsupportedOAuthProviderCount: number; unsupportedOAuthProviders: string[]; }; @@ -207,7 +219,11 @@ export async function resolveDeployState(ctx: DeployContext): Promise descriptor.provider); - const { productionConfig, deployStatus } = await loadProductionState( + const { productionConfig, deployStatus, live } = await loadProductionState( ctx, productionInstanceId, domain.id, @@ -284,6 +300,7 @@ export async function resolveLiveDeploySnapshot( completedOAuthProviders, cnameTargets: domain.cname_targets ?? [], componentStatus: deployComponentStatusFromDomainStatus(deployStatus), + live, unsupportedOAuthProviderCount: unsupported.length, unsupportedOAuthProviders: unsupported, }; @@ -313,17 +330,16 @@ export async function loadInitialDeployStatus( appId: string, domainIdOrName: string, options: SnapshotOptions = {}, -): Promise { - const status = mapDeployError(getApplicationDomainStatus(appId, domainIdOrName)); - if (options.throwOnStatusError) return status; - +): Promise<{ status: DomainStatusResponse; live: boolean }> { try { - return await status; + const status = await mapDeployError(getApplicationDomainStatus(appId, domainIdOrName)); + return { status, live: true }; } catch (error) { + if (options.throwOnStatusError) throw error; log.debug( `deploy: snapshot domain-status read failed, treating DNS as pending: ${error instanceof Error ? error.message : String(error)}`, ); - return pendingDomainStatus(); + return { status: pendingDomainStatus(), live: false }; } } @@ -335,13 +351,14 @@ export async function loadProductionState( ): Promise<{ productionConfig: Record; deployStatus: DomainStatusResponse; + live: boolean; }> { return withSpinner("Reading production configuration...", async () => { - const [productionConfig, deployStatus] = await Promise.all([ + const [productionConfig, { status: deployStatus, live }] = await Promise.all([ fetchInstanceConfig(ctx.appId, productionInstanceId), loadInitialDeployStatus(ctx.appId, domainIdOrName, options), ]); - return { productionConfig, deployStatus }; + return { productionConfig, deployStatus, live }; }); } @@ -404,12 +421,9 @@ function buildDeployStatusFacts( const { snapshot } = state; const componentStatus = outcome?.status ?? snapshot.componentStatus; const domainComplete = outcome ? outcome.verified : snapshot.domainComplete; - const oauthPending = snapshot.oauthProviders.filter( - (provider) => !snapshot.completedOAuthProviders.includes(provider), - ); - const oauthComplete = oauthPending.length === 0; - const complete = domainComplete && oauthComplete; - const reportState = resolveActiveReportState(domainComplete, complete); + const oauthPending = pendingOAuthProviders(snapshot); + const reportState = deployReportState(state, outcome); + const complete = reportState === "complete"; const pendingDnsRecords: DeployStatusReport["pendingDnsRecords"] = !domainComplete ? pendingCnameTargets(snapshot.cnameTargets ?? [], componentStatus).map((target) => ({ @@ -432,7 +446,7 @@ function buildDeployStatusFacts( }, pendingDnsRecords, oauth: { - complete: oauthComplete, + complete: oauthPending.length === 0, configured: [...snapshot.completedOAuthProviders], pending: oauthPending, unsupported: [...snapshot.unsupportedOAuthProviders], @@ -472,10 +486,112 @@ export function buildInterruptedDeployStatusReport(): DeployStatusReport { }); } -function resolveActiveReportState(domainComplete: boolean, complete: boolean): DeployStatusState { - if (complete) return "complete"; +/** The states a deploy with a production domain can be in. */ +export type ActiveDeployStatusState = Extract< + DeployStatusState, + "domain_pending" | "oauth_pending" | "complete" +>; + +export type OAuthSetupFacts = Pick< + DeployOperationState, + "oauthProviders" | "completedOAuthProviders" +>; + +function pendingOAuthProviders(oauth: OAuthSetupFacts): string[] { + return oauth.oauthProviders.filter( + (provider) => !oauth.completedOAuthProviders.includes(provider), + ); +} + +/** + * The one place the three active states are decided, for the report and for + * telemetry alike. `domainComplete` is the domain-status read's own verdict, + * not the three component booleans: all three can be verified while Clerk is + * still finalizing, and that is still `domain_pending`. + */ +export function resolveActiveReportState( + oauth: OAuthSetupFacts, + domainComplete: boolean, +): ActiveDeployStatusState { if (!domainComplete) return "domain_pending"; - return "oauth_pending"; + return pendingOAuthProviders(oauth).length === 0 ? "complete" : "oauth_pending"; +} + +/** + * The state a report for `state` carries. `outcome` is a wait's latest poll + * and overrides the snapshot's domain verdict when present. + */ +export function deployReportState( + state: DeployState, + outcome: DeployStatusOutcome | null, +): Exclude { + if (state.kind !== "active") return state.kind; + return resolveActiveReportState( + state.snapshot, + outcome ? outcome.verified : state.snapshot.domainComplete, + ); +} + +/** + * Record the deploy's state as telemetry's `stage`. Every write goes through + * this function or {@link recordObservedDeployStage}, and every value is a + * report state — what `clerk deploy status` would print for this deploy at + * this moment — so the wizard, the agent handoff and the status command agree + * about the same deploy. + * + * One rule across every writer: last write wins, and nothing is written + * without an observation, so a run that ends before any state is known sends + * null. The writers, in the order a run can reach them: + * + * - `startNewDeploy` on entry: `not_started`. The create call has not run. + * - `startNewDeploy` when the create call finds an instance already exists: + * {@link retractDeployStage}. An instance exists; nothing else is known. + * - `startNewDeploy` on the create response: `domain_pending` or + * `domain_provisioning`, from whether Clerk returned a domain. + * - `reconcileExistingDeploy`: `domain_provisioning` when Clerk lists no + * domain, else the snapshot's state through `recordObservedDeployStage`. + * - `runDnsVerification`, once per poll: that poll's verdict. + * - `finishDeploy`: the resolver over the OAuth facts and a verified domain. + * - `emitAgentDeployHandoff` and `deployStatus`: the state read, then each + * poll, through `recordObservedDeployStage`. + * + * This entry takes a state the caller can vouch for without a snapshot — one + * the CLI's own action established, or a poll's verdict. A state derived from + * a snapshot goes through `recordObservedDeployStage`, which is where the + * substituted-read check lives. `interrupted` is not a state of the deploy: + * it means nothing was read, so whatever was last observed stays in place. + */ +export function recordDeployStage(state: DeployStatusState): void { + if (state === "interrupted") return; + setTelemetryStage(state); +} + +/** + * Record the state a `DeployState` establishes, or nothing when it rests on a + * substituted snapshot. A poll outcome is its own observation, so with one + * present the snapshot's liveness does not matter. This is the only place + * that stops a fallback being recorded as an observation: the callers that + * read through `resolveDeployState` never trip it today, because that read + * throws rather than substitutes, and the check is here so that stays true + * without every call site knowing about the option. + */ +export function recordObservedDeployStage( + state: DeployState, + outcome: DeployStatusOutcome | null, +): void { + if (state.kind === "active" && !outcome && !state.snapshot.live) return; + recordDeployStage(deployReportState(state, outcome)); +} + +/** + * Forget the recorded stage. For the one case where an observation disproves + * the stage without establishing a new one: a fresh deploy's create call + * answering that an instance already exists. `not_started` is now false, and + * whether that instance has a domain, or how far it got, is unknown until the + * resume reads it — and the resume records normally when it does. + */ +export function retractDeployStage(): void { + clearTelemetryStage(); } /** @@ -646,7 +762,7 @@ export async function waitForDeployStatus( } let response = await mapDeployError(getApplicationDomainStatus(appId, domainIdOrName)); let status = deployComponentStatusFromDomainStatus(response); - handlers.onStatus?.(status); + handlers.onStatus?.({ verified: response.status === "complete", status }); const labels = deployComponentLabels("dns", domain); const verified = await handlers.runVerification(labels.progress, async (spinner) => { @@ -666,7 +782,7 @@ export async function waitForDeployStatus( nextRetryDelay *= DEPLOY_STATUS_BACKOFF_FACTOR; response = await mapDeployError(getApplicationDomainStatus(appId, domainIdOrName)); status = deployComponentStatusFromDomainStatus(response); - handlers.onStatus?.(status); + handlers.onStatus?.({ verified: response.status === "complete", status }); if (response.status === "complete") return true; } return false; diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 1c540c469..cf50d152d 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -123,9 +123,10 @@ export type TelemetryStage = // state of the deploy itself, as `resolveActiveReportState` in // `commands/deploy/status.ts` would compute it at that moment. So the stage // a wizard run reports and the stage `clerk deploy status` reports a second - // later agree about the same deploy. The last one set is sent, and a run - // that ends before any state resolves sends null rather than defaulting — - // "never established" is a distinct answer from "not started". + // later agree about the same deploy. One value per run — the last state + // observed, not every state the run passed through — and a run that ends + // before any state resolves sends null rather than defaulting: "never + // established" is a distinct answer from "not started". // // A finished deploy is `complete`, never the shared `done` marker below: // the warehouse's payload contract test accepts exactly these five values @@ -293,6 +294,16 @@ export function setTelemetryStage(stage: TelemetryStage): void { if (context) context.stage = stage; } +/** + * Forget the stage. For the one case where an observation disproves the + * stage last set without establishing a new one — `retractDeployStage` in + * `commands/deploy/status.ts` is the only caller. Not a general reset: a + * command that wants a different stage sets it. + */ +export function clearTelemetryStage(): void { + if (context) context.stage = null; +} + /** Read the stage a caller had set, so a nested flow can hand it back. */ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts index 98eaf5819..975cdf9bf 100644 --- a/packages/cli-core/src/test/integration/telemetry.test.ts +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -156,14 +156,46 @@ test("an unfinished `deploy status` is recorded as incomplete, not an error", as expect(event.payload.command).toBe("deploy status"); expect(event.payload.outcome).toBe("incomplete"); expect(event.payload.exit_code).toBe(1); + expect(event.payload.stage).toBe("not_started"); // Nothing was thrown, so there is no code to carry — the two fields are // unrelated, and `incomplete` is the whole answer. expect(event.payload.error_code).toBeNull(); } finally { - process.exitCode = undefined; + // Bun ignores `process.exitCode = undefined`; only a number resets it. + process.exitCode = 0; } }); +// Drives the real program rather than the unit tests' capture helper, so it +// pins two things only `runProgram` can: the command name Commander gives the +// hidden default subcommand — the warehouse contract keys on `deploy run` — +// and that the stage set inside the command reaches the event it sends. +test("`clerk deploy` under an agent with no production instance records stage not_started", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + await setProfile("github.com/test/project", { + workspaceId: "", + appId: MOCK_APP_DEV_ONLY.application_id, + instances: { development: getInstance(MOCK_APP_DEV_ONLY, "development").instance_id }, + }); + http.mock({ + [`/applications/${MOCK_APP_DEV_ONLY.application_id}`]: MOCK_APP_DEV_ONLY, + "test-telemetry.clerk.com": {}, + }); + + const result = await clerk("--mode", "agent", "deploy"); + + expect(JSON.parse(result.stdout).state).toBe("not_started"); + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("deploy run"); + expect(event.payload.outcome).toBe("success"); + expect(event.payload.exit_code).toBe(0); + expect(event.payload.stage).toBe("not_started"); + expect(event.payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); +}); + test("maps a soft failure (process.exitCode set without throwing) to outcome error", async () => { await markNoticeAlreadyShown(); process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; @@ -180,7 +212,8 @@ test("maps a soft failure (process.exitCode set without throwing) to outcome err expect(event.payload.outcome).toBe("error"); expect(event.payload.exit_code).toBe(1); } finally { - process.exitCode = undefined; + // Bun ignores `process.exitCode = undefined`; only a number resets it. + process.exitCode = 0; } }); diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index cdded260f..d80d15e8d 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -340,7 +340,10 @@ export async function captureTelemetryPayload( process.env.CLERK_TELEMETRY_URL = TELEMETRY_CAPTURE_URL; delete process.env.CLERK_TELEMETRY_DISABLED; delete process.env.DO_NOT_TRACK; - process.exitCode = undefined; + // Not `undefined`: Bun ignores that assignment and keeps the previous + // number, so a run left at 1 by an earlier test would classify every + // later success as an error. 0 classifies exactly as unset does. + process.exitCode = EXIT_CODE.SUCCESS; globalThis.fetch = (async (url: unknown, init?: { body?: string }) => { if (String(url) !== TELEMETRY_CAPTURE_URL) { return savedFetch(url as Parameters[0], init as RequestInit); @@ -385,7 +388,7 @@ export async function captureTelemetryPayload( return { payload: parsed.events[0]!.payload, error }; } finally { globalThis.fetch = savedFetch; - process.exitCode = savedExitCode; + process.exitCode = savedExitCode ?? EXIT_CODE.SUCCESS; for (const [key, value] of Object.entries(savedEnv)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; From a2111b70296785d3c090ee29e03f16dbd84d272e Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 09:20:21 -0700 Subject: [PATCH 06/16] Record which deploy components each run observed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `clerk deploy` and `clerk deploy status` event now carries four booleans — DNS, SSL, email DNS and OAuth — each `true`, `false` or null, where null means no successful read ever established it. A failed status call is not a DNS failure, so nothing is written from a substituted or placeholder status. The four come from two reads, so they are two setters. DNS, SSL and mail come from the domain-status response and OAuth from whether every required provider has production credentials. A domain poll rewrites the first three and leaves `oauth` as it was, because the poll did not observe it; a configuration read or a credential save rewrites `oauth` alone. A single setter for all four would discard a good OAuth observation whenever the domain read flaked, and every later poll would have to re-send or blank it. The case this exists for: the wizard's resume path substitutes "everything pending" when its domain read fails so the user can retry from the screen. That snapshot now records `oauth` from the configuration read that did succeed, and nothing for the domain — one observation, not three false values for a network blip. --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/deploy/README.md | 2 + .../src/commands/deploy/index.test.ts | 63 ++++++++++ .../cli-core/src/commands/deploy/index.ts | 26 +++- .../commands/deploy/status-command.test.ts | 115 ++++++++++++++++++ .../src/commands/deploy/status-command.ts | 7 +- .../src/commands/deploy/status.test.ts | 98 ++++++++++----- .../cli-core/src/commands/deploy/status.ts | 88 +++++++++----- packages/cli-core/src/lib/telemetry.test.ts | 53 ++++++++ packages/cli-core/src/lib/telemetry.ts | 34 ++++++ 10 files changed, 419 insertions(+), 69 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 51fa98062..9006df813 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -2,6 +2,6 @@ "clerk": patch --- -Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Every `clerk deploy` and `clerk deploy status` event now also records the state the deploy was in when the run ended, so a run that stopped short says where. Output and exit codes are unchanged. +Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Every `clerk deploy` and `clerk deploy status` event now also records the state the deploy was in when the run ended, so a run that stopped short says where, and which of DNS, SSL, email DNS and OAuth had been verified at that point — recorded only from a read that actually succeeded, so a failed status call is never reported as a failed check. Output and exit codes are unchanged. `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index f7290ef80..74d76cc4c 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -83,6 +83,8 @@ Four ways a run ends with the deploy unfinished and nothing broken. All four exi It is null when no reliable state was established by the time the run ended, and that null is a different answer from `not_started`. That covers a run that failed before reading anything — not linked, a failed sign-in, an API error on the first read — and two cases where a state was invalidated or never observed: a resume whose domain read failed and substituted an all-pending status so the user could retry from the screen, where the user then skipped verification; and a fresh run whose create call answered that an instance already exists, after which the resume could not read it. Two states are known without a status read: a fresh deploy starts at `not_started`, and a newly created instance is at `domain_pending` the moment Clerk returns it with a domain (`domain_provisioning` if it did not). Every other value comes from a read that succeeded. +`components` says which of the four pieces were verified when the run ended: `dns`, `ssl` and `mail` from the domain-status response, `oauth` from whether every required provider has production credentials — the same facts the status report's `domainStatus` and `oauth.complete` print. Each is `true`, `false` or null, and null means never observed, which is a different answer from `false`: a failed status call is not a DNS failure. The four come from two reads, so they are two observations. A domain poll rewrites the first three and leaves `oauth` as it was; a production-configuration read or a credential save rewrites `oauth` and leaves the other three. Within a group the last observation wins. A read that never happened, or a substituted one, writes nothing — so a resume whose domain read failed sends `oauth` from its configuration read and null for the other three, and a fresh run that saves every provider's credentials and skips DNS verification sends `oauth: true` with the other three null. `oauth` reflects the CLI's required-provider rule as it stands; when GROW-1236 changes that rule, this value follows, because it is computed from the same report. + Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: 1. `--mode` CLI flag diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index ec075ff9f..0db5bbc60 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -2674,6 +2674,10 @@ describe("deploy", () => { expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_PAUSED); expect(payload.pause_step).toBe("oauth"); expect(payload.stage).toBe("domain_pending"); + // Nothing was read and nothing was saved, so no component is observed + // — not even OAuth, which the wizard knows is unconfigured but has + // not read from the production configuration. + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); // The same deploy, read back: instance present, DNS unverified, OAuth // unconfigured. @@ -2686,6 +2690,12 @@ describe("deploy", () => { expect(status.payload.stage).toBe("domain_pending"); expect(status.payload.outcome).toBe("incomplete"); + expect(status.payload.components).toEqual({ + dns: false, + ssl: false, + mail: false, + oauth: false, + }); }); test("Ctrl-C at the DNS retry prompt is a cancelled deploy at the dns step", async () => { @@ -2994,6 +3004,7 @@ describe("deploy", () => { expect(payload.outcome).toBe("success"); expect(payload.stage).toBe("complete"); + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); }); test("completing the last OAuth provider on a verified domain ends at complete", async () => { @@ -3008,6 +3019,50 @@ describe("deploy", () => { expect(payload.outcome).toBe("success"); expect(payload.stage).toBe("complete"); + // The domain group from the live resume read; oauth from the save that + // followed it, overriding the `false` the read observed. + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); + }); + + test("a fresh run that saves every provider and skips DNS observes oauth alone", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockOAuthCompletion(); + mockSelect.mockResolvedValueOnce("skip"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("domain_pending"); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); + }); + + test("a failed poll after a live resume read keeps what the read observed", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockGetApplicationDomainStatus + .mockResolvedValueOnce( + domainStatus({ status: "incomplete", dns: false, ssl: true, mail: true }), + ) + .mockRejectedValue( + new PlapiError( + 500, + JSON.stringify({ errors: [{ code: "server_error" }] }), + "https://x", + ), + ); + mockConfirm.mockResolvedValueOnce(false); + mockSelect.mockResolvedValueOnce("check"); + + const { payload, error } = await deployTelemetry(async () => runDeploy({})); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.stage).toBe("domain_pending"); + expect(payload.components).toEqual({ dns: false, ssl: true, mail: true, oauth: true }); }); // The resume path substitutes "everything pending" when the domain-status @@ -3029,6 +3084,13 @@ describe("deploy", () => { expect(payload.outcome).toBe("success"); expect(payload.stage).toBeNull(); + // The configuration read succeeded and the domain read did not: one + // observation, not four falses for a network blip. + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); + // And the user still gets the normal retry screen. + expect(mockSelect).toHaveBeenCalledWith( + expect.objectContaining({ message: "DNS verification" }), + ); }); test("a resume whose domain read failed records what a later successful poll observes", async () => { @@ -3054,6 +3116,7 @@ describe("deploy", () => { expect(payload.outcome).toBe("success"); expect(payload.stage).toBe("complete"); + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); }); // The stage is recorded per poll, not once the wait returns: the signal diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index e1315bed4..525ec77b7 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -65,8 +65,10 @@ import { import { buildDeployStatusReport, loadDevelopmentOAuthProviders, + recordDeployObservation, + recordDeployPoll, recordDeployStage, - recordObservedDeployStage, + recordOAuthObservation, resolveActiveReportState, resolveDeployContext, resolveDeployState, @@ -130,7 +132,7 @@ async function emitAgentDeployHandoff(): Promise { } const state = await resolveDeployState(ctx); - recordObservedDeployStage(state, null); + recordDeployObservation(state); const report = buildDeployStatusReport(state, null); log.data(JSON.stringify(report, null, 2)); } @@ -289,9 +291,10 @@ async function reconcileExistingDeploy(ctx: DeployContext): Promise { await outro("No deploy actions available"); return; } - // Records nothing when the domain read was substituted; a later poll that - // succeeds will. - recordObservedDeployStage({ kind: "active", snapshot }, null); + // Records no stage or domain components when the domain read was + // substituted; a later poll that succeeds will. OAuth is recorded either + // way, since the configuration read succeeded. + recordDeployObservation({ kind: "active", snapshot }); log.blank(); for (const line of printPlan(ctx.appLabel, buildLiveDeployPlan(snapshot))) { @@ -527,7 +530,7 @@ async function runDnsVerification( // Per poll, not once the wait returns: a Ctrl-C mid-wait is reported by // the signal handler with the stage as it stands at that moment. const outcome = await pollDeployStatus(ctx.appId, domainIdOrName, state.domain, (polled) => - recordDeployStage(resolveActiveReportState(state, polled.verified)), + recordDeployPoll(state, polled), ); if (outcome.verified) { @@ -620,6 +623,9 @@ async function runOAuthSetup( descriptors: readonly OAuthProviderDescriptor[], ): Promise { const completed = new Set(state.completedOAuthProviders as OAuthProvider[]); + const oauthProviders = descriptors.map((descriptor) => descriptor.provider); + const recordOAuth = () => + recordOAuthObservation({ oauthProviders, completedOAuthProviders: [...completed] }); if (descriptors.length > 0) { log.info(OAUTH_SECTION_INTRO); @@ -670,11 +676,19 @@ async function runOAuthSetup( throw error; } completed.add(descriptor.provider); + // Each save is an observation of the production configuration: this + // provider now has credentials, the ones after it still do not. A pause + // on the next provider then reports `oauth: false` from a real write, not + // from a guess. + recordOAuth(); if (descriptors.some((nextDescriptor) => !completed.has(nextDescriptor.provider))) { log.blank(); } } + // Also the deploy with nothing to configure: no provider is required, so + // OAuth is complete, which is what `deploy status` reports for it too. + recordOAuth(); return [...completed]; } diff --git a/packages/cli-core/src/commands/deploy/status-command.test.ts b/packages/cli-core/src/commands/deploy/status-command.test.ts index 97b01f3a0..86b9bdc6b 100644 --- a/packages/cli-core/src/commands/deploy/status-command.test.ts +++ b/packages/cli-core/src/commands/deploy/status-command.test.ts @@ -950,6 +950,121 @@ describe("deploy status", () => { expect(error).toBeInstanceOf(PlapiError); expect(payload.stage).toBeNull(); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + }); + + // The four readiness booleans, each from the read that observed it. DNS, + // SSL and email DNS come from the domain-status response; OAuth from the + // production configuration. Null means never observed, and is never + // written as false by a failed read. + describe("components", () => { + function statusTelemetry(options: Parameters[0] = {}) { + return captureTelemetryPayload("deploy status", () => deployStatus(options), { + captureError: true, + }); + } + + function pendingMailDomainStatus() { + return { + status: "incomplete", + dns: { status: "complete" }, + ssl: { status: "complete", required: true }, + mail: { status: "pending", required: true }, + }; + } + + function allPendingDomainStatus() { + return { + status: "incomplete", + dns: { status: "not_started" }, + ssl: { status: "not_started", required: true }, + mail: { status: "not_started", required: true }, + }; + } + + const combinations = [ + ["DNS only", pendingDnsDomainStatus, { dns: false, ssl: true, mail: true, oauth: true }], + ["SSL only", pendingSslDomainStatus, { dns: true, ssl: false, mail: true, oauth: true }], + [ + "email DNS only", + pendingMailDomainStatus, + { dns: true, ssl: true, mail: false, oauth: true }, + ], + [ + "every domain component", + allPendingDomainStatus, + { dns: false, ssl: false, mail: false, oauth: true }, + ], + ["nothing", completeDomainStatus, { dns: true, ssl: true, mail: true, oauth: true }], + ] as const; + + for (const [pending, domainStatus, expected] of combinations) { + test(`${pending} pending records the observed booleans`, async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(domainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(domainStatus()); + + const { payload } = await statusTelemetry(); + + expect(payload.components).toEqual(expected); + }); + } + + test("OAuth only pending records oauth false with the domain verified", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockFetchInstanceConfig.mockImplementation(() => ({ + connection_oauth_google: { enabled: true }, + })); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(completeDomainStatus()); + mockGetApplicationDomainStatus.mockResolvedValue(completeDomainStatus()); + + const { payload } = await statusTelemetry(); + + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: false }); + }); + + test("polls under --wait update the domain group and leave oauth as first observed", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + mockGetApplicationDomainStatus + .mockResolvedValueOnce(pendingDnsDomainStatus()) + .mockResolvedValueOnce(pendingSslDomainStatus()) + .mockResolvedValue(completeDomainStatus()); + + const { payload } = await statusTelemetry({ wait: true }); + + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); + // Development and production configuration, once each: no poll re-reads OAuth. + expect(mockFetchInstanceConfig).toHaveBeenCalledTimes(2); + }); + + test("a failed poll after a successful state read keeps what the read observed", async () => { + mockFetchApplication.mockResolvedValue(appWith(true)); + mockDomain(); + mockOAuthComplete(); + mockTriggerApplicationDomainDNSCheck.mockResolvedValue(pendingDnsDomainStatus()); + mockGetApplicationDomainStatus + .mockResolvedValueOnce(pendingDnsDomainStatus()) + .mockRejectedValue( + new PlapiError( + 500, + JSON.stringify({ errors: [{ code: "server_error" }] }), + "https://x", + ), + ); + + const { payload, error } = await statusTelemetry({ wait: true }); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.stage).toBe("domain_pending"); + expect(payload.components).toEqual({ dns: false, ssl: true, mail: true, oauth: true }); }); }); }); diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index a9a29c1ac..fc6ba204b 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -11,7 +11,8 @@ import { buildInterruptedDeployStatusReport, deployNextStep, loadProductionDomain, - recordObservedDeployStage, + recordDeployObservation, + recordDeployPoll, resolveDeployContext, resolveDeployState, triggerDeployStatusCheck, @@ -57,7 +58,7 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { lastPolled = polled; - recordObservedDeployStage(active, polled); + recordDeployPoll(active.snapshot, polled); }, }); } diff --git a/packages/cli-core/src/commands/deploy/status.test.ts b/packages/cli-core/src/commands/deploy/status.test.ts index 7be255bb8..e409553c7 100644 --- a/packages/cli-core/src/commands/deploy/status.test.ts +++ b/packages/cli-core/src/commands/deploy/status.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { PlapiError } from "../../lib/errors.ts"; -import { fakeTelemetryCommand } from "../../test/lib/stubs.ts"; +import { captureTelemetryPayload } from "../../test/lib/stubs.ts"; import type { LiveDeploySnapshot } from "./status.ts"; const mockFetchApplication = mock(); @@ -26,12 +29,13 @@ const { buildInterruptedDeployStatusReport, deployNextStep, loadInitialDeployStatus, - recordObservedDeployStage, + recordDeployObservation, + recordDeployPoll, resolveDeployState, waitForDeployStatus, } = await import("./status.ts"); -const { currentTelemetryStage, setTelemetryStage, startCommandTelemetry } = - await import("../../lib/telemetry.ts"); +const { setTelemetryStage } = await import("../../lib/telemetry.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); const ctx = { profileKey: "/tmp/x", @@ -245,8 +249,9 @@ describe("loadInitialDeployStatus", () => { // The only guard between a substituted "everything pending" read and a // recorded DNS stall. The status command never trips it today because its // read throws instead of substituting; the guard is here so that a change to -// that read cannot silently start recording fallbacks. -describe("recordObservedDeployStage", () => { +// that read cannot silently start recording fallbacks. Read back through the +// posted payload, since that is the only place the components are visible. +describe("recording observations", () => { const snapshot = { appId: "app_1", developmentInstanceId: "ins_dev", @@ -259,48 +264,81 @@ describe("recordObservedDeployStage", () => { cnameTargets: [], domainComplete: false, live: true, - componentStatus: { dns: false, ssl: false, mail: false }, + componentStatus: { dns: false, ssl: true, mail: true }, unsupportedOAuthProviderCount: 0, unsupportedOAuthProviders: [], pending: { type: "dns" as const }, } satisfies LiveDeploySnapshot; + let tempDir = ""; - beforeEach(() => { - startCommandTelemetry(fakeTelemetryCommand("deploy status")); + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), "clerk-status-observe-")); + _setConfigDir(tempDir); }); - test("a live snapshot records its state", () => { - recordObservedDeployStage({ kind: "active", snapshot }, null); - - expect(currentTelemetryStage()).toBe("domain_pending"); + afterEach(async () => { + _setConfigDir(undefined); + await rm(tempDir, { recursive: true, force: true }); }); - test("a substituted snapshot leaves the stage as it was", () => { - setTelemetryStage("oauth_pending"); + async function recorded(run: () => void) { + const { payload } = await captureTelemetryPayload("deploy status", run, { + result: { outcome: "success", exitCode: 0 }, + }); + return { stage: payload.stage, components: payload.components }; + } - recordObservedDeployStage({ kind: "active", snapshot: { ...snapshot, live: false } }, null); + test("a live snapshot records its state and all four components", async () => { + const result = await recorded(() => recordDeployObservation({ kind: "active", snapshot })); - expect(currentTelemetryStage()).toBe("oauth_pending"); + expect(result).toEqual({ + stage: "domain_pending", + components: { dns: false, ssl: true, mail: true, oauth: true }, + }); }); - test("a poll outcome is its own observation, whatever the snapshot was", () => { - recordObservedDeployStage( - { kind: "active", snapshot: { ...snapshot, live: false } }, - { verified: true, status: { dns: true, ssl: true, mail: true } }, - ); + // The configuration read succeeded, so OAuth is an observation; the domain + // read did not, so nothing about the domain is, and the stage stays as it was. + test("a substituted snapshot records OAuth only", async () => { + const result = await recorded(() => { + setTelemetryStage("oauth_pending"); + recordDeployObservation({ kind: "active", snapshot: { ...snapshot, live: false } }); + }); + + expect(result).toEqual({ + stage: "oauth_pending", + components: { dns: null, ssl: null, mail: null, oauth: true }, + }); + }); + + test("a poll records the domain group and stage, and leaves OAuth as observed", async () => { + const result = await recorded(() => { + recordDeployObservation({ kind: "active", snapshot: { ...snapshot, live: false } }); + recordDeployPoll(snapshot, { verified: true, status: { dns: true, ssl: true, mail: true } }); + }); - expect(currentTelemetryStage()).toBe("complete"); + expect(result).toEqual({ + stage: "complete", + components: { dns: true, ssl: true, mail: true, oauth: true }, + }); }); - test("the two states without a snapshot record directly", () => { - recordObservedDeployStage({ kind: "not_started" }, null); - expect(currentTelemetryStage()).toBe("not_started"); + test("the two states without a snapshot record a stage and no components", async () => { + const notStarted = await recorded(() => recordDeployObservation({ kind: "not_started" })); + expect(notStarted).toEqual({ + stage: "not_started", + components: { dns: null, ssl: null, mail: null, oauth: null }, + }); - recordObservedDeployStage( - { kind: "domain_provisioning", appId: "app_1", productionInstanceId: "ins_prod" }, - null, + const provisioning = await recorded(() => + recordDeployObservation({ + kind: "domain_provisioning", + appId: "app_1", + productionInstanceId: "ins_prod", + }), ); - expect(currentTelemetryStage()).toBe("domain_provisioning"); + expect(provisioning.stage).toBe("domain_provisioning"); + expect(provisioning.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); }); }); diff --git a/packages/cli-core/src/commands/deploy/status.ts b/packages/cli-core/src/commands/deploy/status.ts index 2d9bd6f02..ceef5b7c1 100644 --- a/packages/cli-core/src/commands/deploy/status.ts +++ b/packages/cli-core/src/commands/deploy/status.ts @@ -33,7 +33,12 @@ import { type OAuthProviderDescriptor, } from "./providers.ts"; import type { DeployContext, DeployOperationState } from "./state.ts"; -import { clearTelemetryStage, setTelemetryStage } from "../../lib/telemetry.ts"; +import { + clearTelemetryStage, + setTelemetryDomainComponents, + setTelemetryOAuthComplete, + setTelemetryStage, +} from "../../lib/telemetry.ts"; const DEPLOY_STATUS_INITIAL_RETRY_DELAY_MS = 3000; const DEPLOY_STATUS_MAX_RETRIES = 5; @@ -220,8 +225,8 @@ export async function resolveDeployState(ctx: DeployContext): Promise { }); }); + // Four booleans from two reads. Each setter owns its group and must not + // touch the other: a domain poll that re-sent OAuth would either blank a + // good observation or repeat a stale one. + describe("components", () => { + const success = { outcome: "success" as const, exitCode: 0 }; + + test("null until observed", async () => { + const payload = await sendAndCapturePayload(() => {}, success); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + + test("the domain setter leaves oauth alone", async () => { + const payload = await sendAndCapturePayload( + () => setTelemetryDomainComponents({ dns: true, ssl: false, mail: true }), + success, + ); + expect(payload.components).toEqual({ dns: true, ssl: false, mail: true, oauth: null }); + }); + + test("the oauth setter leaves the domain group alone", async () => { + const payload = await sendAndCapturePayload(() => setTelemetryOAuthComplete(false), success); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: false }); + }); + + test("within a group the last write wins, across groups each keeps its own", async () => { + const payload = await sendAndCapturePayload(() => { + setTelemetryOAuthComplete(true); + setTelemetryDomainComponents({ dns: false, ssl: false, mail: false }); + setTelemetryDomainComponents({ dns: true, ssl: false, mail: true }); + }, success); + expect(payload.components).toEqual({ dns: true, ssl: false, mail: true, oauth: true }); + }); + + test("does not leak into the next run", async () => { + await sendAndCapturePayload(() => { + setTelemetryOAuthComplete(true); + setTelemetryDomainComponents({ dns: true, ssl: true, mail: true }); + }, success); + + const payload = await sendAndCapturePayload(() => {}, success); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + + test("setting with no active context is a no-op", () => { + expect(() => + setTelemetryDomainComponents({ dns: true, ssl: true, mail: true }), + ).not.toThrow(); + expect(() => setTelemetryOAuthComplete(true)).not.toThrow(); + }); + }); + // A command that reports failure through `process.exitCode` never reaches // `telemetryResultForError`, so without a declaration the only thing the // soft-exit branch can say is "nonzero, therefore error". diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index cf50d152d..e3efa98aa 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -323,6 +323,40 @@ export function setTelemetryPauseStep(step: TelemetryPauseStep): void { if (context) context.pauseStep = step; } +/** + * Record what a successful domain-status read said about DNS, SSL and email + * DNS. Only ever called with a live read's answer: the wizard's substituted + * "everything pending" status and its fresh-run placeholder are not + * observations, and recording either would file a network blip as a DNS + * failure. Leaves `oauth` alone — it comes from a different read, and a + * domain poll must not erase a good OAuth observation or re-send a stale one. + */ +export function setTelemetryDomainComponents(status: { + dns: boolean; + ssl: boolean; + mail: boolean; +}): void { + if (!context) return; + context.components = { + ...context.components, + dns: status.dns, + ssl: status.ssl, + mail: status.mail, + }; +} + +/** + * Record whether every required OAuth provider has production credentials, + * from a successful production-configuration read or a credential save. + * "Required" is the CLI's rule as it stands — the providers enabled in + * development that the wizard knows how to configure. GROW-1236 changes that + * rule to read production configuration; this value follows automatically, + * because it is computed from the same report. Leaves the domain group alone. + */ +export function setTelemetryOAuthComplete(complete: boolean): void { + if (context) context.components = { ...context.components, oauth: complete }; +} + /** * Declare what this run should be recorded as when it ends by setting * `process.exitCode` instead of throwing. From b8dfcf49b8ef5686327330690c8f82a4a89b20bd Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 09:56:57 -0700 Subject: [PATCH 07/16] Record each deploy read the moment it succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production configuration and domain status are read together, and until now nothing was written to telemetry until both had answered. A 500 from the domain-status endpoint while the configuration read was fine — the ordinary shape of a partial outage — ended the run with all four component fields null after one read had observed something. Each read now records its own group the instant it succeeds, inside `resolveLiveDeploySnapshot`. The error the user sees is unchanged: the first failure still wins. The throw is only delayed until the partner read has settled, so the recording can never race the telemetry send. The stage still needs both reads and stays with `recordDeployObservation`. Also drops the per-save OAuth write. A save proves only the provider it saved, and recording `false` for the rest asserted that a cloned instance carries no credentials — an assumption the wizard makes but nothing has verified. OAuth is now written from a configuration read, or as `true` once every required credential is saved, which is the plan's rule. --- .../cli-core/src/commands/deploy/README.md | 2 +- .../src/commands/deploy/index.test.ts | 30 ++++ .../cli-core/src/commands/deploy/index.ts | 18 +-- .../commands/deploy/status-command.test.ts | 9 +- .../src/commands/deploy/status.test.ts | 147 +++++++++++++++--- .../cli-core/src/commands/deploy/status.ts | 138 ++++++++++------ 6 files changed, 261 insertions(+), 83 deletions(-) diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 74d76cc4c..43d6952c9 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -83,7 +83,7 @@ Four ways a run ends with the deploy unfinished and nothing broken. All four exi It is null when no reliable state was established by the time the run ended, and that null is a different answer from `not_started`. That covers a run that failed before reading anything — not linked, a failed sign-in, an API error on the first read — and two cases where a state was invalidated or never observed: a resume whose domain read failed and substituted an all-pending status so the user could retry from the screen, where the user then skipped verification; and a fresh run whose create call answered that an instance already exists, after which the resume could not read it. Two states are known without a status read: a fresh deploy starts at `not_started`, and a newly created instance is at `domain_pending` the moment Clerk returns it with a domain (`domain_provisioning` if it did not). Every other value comes from a read that succeeded. -`components` says which of the four pieces were verified when the run ended: `dns`, `ssl` and `mail` from the domain-status response, `oauth` from whether every required provider has production credentials — the same facts the status report's `domainStatus` and `oauth.complete` print. Each is `true`, `false` or null, and null means never observed, which is a different answer from `false`: a failed status call is not a DNS failure. The four come from two reads, so they are two observations. A domain poll rewrites the first three and leaves `oauth` as it was; a production-configuration read or a credential save rewrites `oauth` and leaves the other three. Within a group the last observation wins. A read that never happened, or a substituted one, writes nothing — so a resume whose domain read failed sends `oauth` from its configuration read and null for the other three, and a fresh run that saves every provider's credentials and skips DNS verification sends `oauth: true` with the other three null. `oauth` reflects the CLI's required-provider rule as it stands; when GROW-1236 changes that rule, this value follows, because it is computed from the same report. +`components` says which of the four pieces were verified when the run ended: `dns`, `ssl` and `mail` from the domain-status response, `oauth` from whether every required provider has production credentials — the same facts the status report's `domainStatus` and `oauth.complete` print. Each is `true`, `false` or null, and null means never observed, which is a different answer from `false`: a failed status call is not a DNS failure. The four come from two reads, so they are two observations. A domain poll rewrites the first three and leaves `oauth` as it was; a production-configuration read or a credential save rewrites `oauth` and leaves the other three. Within a group the last observation wins. A read that never happened, or a substituted one, writes nothing, and each read is recorded the moment it succeeds, so a failure in the other read does not discard it. So a resume whose domain read failed sends `oauth` from its configuration read and null for the other three; a `clerk deploy status` whose domain read failed does the same, with a null stage, since no state was established. On a fresh deploy, `oauth` is written only once every required credential is saved (`true`), because until then nothing has read the production configuration — a save proves only the provider it saved. A fresh run that pauses part-way through OAuth setup therefore sends null for `oauth` even though `clerk deploy status` on the same deploy would read the configuration and say `false`; that is the one place the two are allowed to differ, and null rather than an assumed `false` is deliberate. A fresh run that saves every provider's credentials and skips DNS verification sends `oauth: true` with the other three null. `oauth` reflects the CLI's required-provider rule as it stands; when GROW-1236 changes that rule, this value follows, because it is computed from the same report. Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 0db5bbc60..443f3cf9f 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -3024,6 +3024,36 @@ describe("deploy", () => { expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); }); + // A save proves only the provider it saved. Until every required one is + // saved nothing has read the production configuration, so a fresh run + // that pauses mid-way sends null — `deploy status` on the same deploy + // would read it and say false, and null is the documented exception. + test("a fresh run that saves one provider and skips the next observes nothing yet", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockFetchInstanceConfig.mockResolvedValue({ + connection_oauth_google: { enabled: true }, + connection_oauth_github: { enabled: true }, + }); + mockFetchInstanceConfigSchema.mockResolvedValue( + schemaResponse({ + connection_oauth_google: basicOAuthSchema, + connection_oauth_github: basicOAuthSchema, + }), + ); + mockConfirm.mockResolvedValueOnce(true).mockResolvedValueOnce(true); + mockInput.mockResolvedValueOnce("example.com"); + mockOAuthCompletion(); + mockSelect.mockResolvedValueOnce("skip"); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_PAUSED); + expect(payload.pause_step).toBe("oauth"); + expect(payload.stage).toBe("domain_pending"); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + test("a fresh run that saves every provider and skips DNS observes oauth alone", async () => { await linkedProject(); mockIsAgent.mockReturnValue(false); diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index 525ec77b7..eb0ee9b91 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -624,8 +624,6 @@ async function runOAuthSetup( ): Promise { const completed = new Set(state.completedOAuthProviders as OAuthProvider[]); const oauthProviders = descriptors.map((descriptor) => descriptor.provider); - const recordOAuth = () => - recordOAuthObservation({ oauthProviders, completedOAuthProviders: [...completed] }); if (descriptors.length > 0) { log.info(OAUTH_SECTION_INTRO); @@ -676,19 +674,19 @@ async function runOAuthSetup( throw error; } completed.add(descriptor.provider); - // Each save is an observation of the production configuration: this - // provider now has credentials, the ones after it still do not. A pause - // on the next provider then reports `oauth: false` from a real write, not - // from a guess. - recordOAuth(); if (descriptors.some((nextDescriptor) => !completed.has(nextDescriptor.provider))) { log.blank(); } } - // Also the deploy with nothing to configure: no provider is required, so - // OAuth is complete, which is what `deploy status` reports for it too. - recordOAuth(); + // Every required credential is saved — including when none is required — + // so OAuth is complete, which is what `deploy status` reports for it too. + // Not recorded any earlier: a fresh instance is assumed to have no + // production credentials, which is why this prompts for each provider, but + // nothing has read that, and a save proves only the provider it saved. A run + // that pauses in the loop leaves `oauth` as the last read observed — null + // on a fresh deploy. + recordOAuthObservation({ oauthProviders, completedOAuthProviders: [...completed] }); return [...completed]; } diff --git a/packages/cli-core/src/commands/deploy/status-command.test.ts b/packages/cli-core/src/commands/deploy/status-command.test.ts index 86b9bdc6b..fc209e9f0 100644 --- a/packages/cli-core/src/commands/deploy/status-command.test.ts +++ b/packages/cli-core/src/commands/deploy/status-command.test.ts @@ -938,7 +938,9 @@ describe("deploy status", () => { expect(payload.stage).toBeNull(); }); - test("a failed state read records null, not the state it was about to read", async () => { + // No state was established, so no stage — but the configuration read + // that ran alongside the failed domain read did observe OAuth. + test("a failed domain read records no stage and keeps the OAuth it did observe", async () => { mockFetchApplication.mockResolvedValue(appWith(true)); mockDomain(); mockOAuthComplete(); @@ -950,7 +952,7 @@ describe("deploy status", () => { expect(error).toBeInstanceOf(PlapiError); expect(payload.stage).toBeNull(); - expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); }); }); @@ -1041,7 +1043,8 @@ describe("deploy status", () => { const { payload } = await statusTelemetry({ wait: true }); expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: true }); - // Development and production configuration, once each: no poll re-reads OAuth. + // Guards against a future per-poll configuration read. Unrelated to why + // polls leave `oauth` alone, which is that they never observed it. expect(mockFetchInstanceConfig).toHaveBeenCalledTimes(2); }); diff --git a/packages/cli-core/src/commands/deploy/status.test.ts b/packages/cli-core/src/commands/deploy/status.test.ts index e409553c7..1e9c94bf0 100644 --- a/packages/cli-core/src/commands/deploy/status.test.ts +++ b/packages/cli-core/src/commands/deploy/status.test.ts @@ -32,10 +32,12 @@ const { recordDeployObservation, recordDeployPoll, resolveDeployState, + resolveLiveDeploySnapshot, waitForDeployStatus, } = await import("./status.ts"); const { setTelemetryStage } = await import("../../lib/telemetry.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); +const { beginInterrupt, _resetInterruptState } = await import("../../lib/signals.ts"); const ctx = { profileKey: "/tmp/x", @@ -288,38 +290,32 @@ describe("recording observations", () => { return { stage: payload.stage, components: payload.components }; } - test("a live snapshot records its state and all four components", async () => { + test("a live snapshot records its state", async () => { const result = await recorded(() => recordDeployObservation({ kind: "active", snapshot })); - expect(result).toEqual({ - stage: "domain_pending", - components: { dns: false, ssl: true, mail: true, oauth: true }, - }); + expect(result.stage).toBe("domain_pending"); }); - // The configuration read succeeded, so OAuth is an observation; the domain - // read did not, so nothing about the domain is, and the stage stays as it was. - test("a substituted snapshot records OAuth only", async () => { - const result = await recorded(() => { - setTelemetryStage("oauth_pending"); - recordDeployObservation({ kind: "active", snapshot: { ...snapshot, live: false } }); - }); + test("a poll records the domain group and the stage, and leaves OAuth alone", async () => { + const result = await recorded(() => + recordDeployPoll(snapshot, { verified: true, status: { dns: true, ssl: true, mail: true } }), + ); expect(result).toEqual({ - stage: "oauth_pending", - components: { dns: null, ssl: null, mail: null, oauth: true }, + stage: "complete", + components: { dns: true, ssl: true, mail: true, oauth: null }, }); }); - test("a poll records the domain group and stage, and leaves OAuth as observed", async () => { + test("a substituted snapshot records no stage; its components were the read's to record", async () => { const result = await recorded(() => { + setTelemetryStage("oauth_pending"); recordDeployObservation({ kind: "active", snapshot: { ...snapshot, live: false } }); - recordDeployPoll(snapshot, { verified: true, status: { dns: true, ssl: true, mail: true } }); }); expect(result).toEqual({ - stage: "complete", - components: { dns: true, ssl: true, mail: true, oauth: true }, + stage: "oauth_pending", + components: { dns: null, ssl: null, mail: null, oauth: null }, }); }); @@ -342,6 +338,121 @@ describe("recording observations", () => { }); }); +// Production configuration and domain status are read together, and each is +// recorded the moment it succeeds: a failure in one must not discard what the +// other observed, and the recording must not race the telemetry send. +describe("resolveLiveDeploySnapshot records each read as it succeeds", () => { + const serverError = () => + new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"); + let tempDir = ""; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), "clerk-status-reads-")); + _setConfigDir(tempDir); + mockListApplicationDomains.mockResolvedValue({ + data: [ + { + object: "domain", + id: "dmn_1", + name: "example.com", + is_satellite: false, + is_provider_domain: false, + frontend_api_url: "https://clerk.example.com", + accounts_portal_url: "https://accounts.example.com", + development_origin: "", + cname_targets: [], + }, + ], + total_count: 1, + }); + mockFetchInstanceConfigSchema.mockResolvedValue({ + properties: { + connection_oauth_google: { + type: "object", + properties: { + enabled: { type: "boolean" }, + client_id: { type: "string" }, + client_secret: { type: "string", "x-clerk-sensitive": true }, + }, + }, + }, + }); + }); + + afterEach(async () => { + _resetInterruptState(); + _setConfigDir(undefined); + await rm(tempDir, { recursive: true, force: true }); + }); + + /** Development enabled Google; production has credentials, or the read fails. */ + function mockConfigReads(production: Record | Error) { + mockFetchInstanceConfig.mockImplementation((_appId: string, instanceId: string) => { + if (instanceId !== "ins_prod") return { connection_oauth_google: { enabled: true } }; + return production instanceof Error ? Promise.reject(production) : production; + }); + } + + const configured = { + connection_oauth_google: { enabled: true, client_id: "id", client_secret: "s" }, + }; + + async function resolved(options: { throwOnStatusError?: boolean } = {}) { + return captureTelemetryPayload( + "deploy status", + async () => { + await resolveLiveDeploySnapshot({ ...ctx, productionInstanceId: "ins_prod" }, options); + }, + { captureError: true }, + ); + } + + test("a failed configuration read keeps what the domain read observed", async () => { + mockConfigReads(serverError()); + mockGetApplicationDomainStatus.mockResolvedValue(completeStatus); + + const { payload, error } = await resolved(); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.components).toEqual({ dns: true, ssl: true, mail: true, oauth: null }); + expect(payload.stage).toBeNull(); + }); + + test("on the strict path a failed domain read keeps what the configuration read observed", async () => { + mockConfigReads(configured); + mockGetApplicationDomainStatus.mockRejectedValue(serverError()); + + const { payload, error } = await resolved({ throwOnStatusError: true }); + + expect(error).toBeInstanceOf(PlapiError); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); + expect(payload.stage).toBeNull(); + }); + + test("on the lenient path a failed domain read substitutes and records OAuth alone", async () => { + mockConfigReads(configured); + mockGetApplicationDomainStatus.mockRejectedValue(serverError()); + + const { payload, error } = await resolved(); + + expect(error).toBeUndefined(); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); + }); + + test("a read that completed before an interrupt is kept", async () => { + mockConfigReads(configured); + mockGetApplicationDomainStatus.mockImplementation(() => { + beginInterrupt(); + throw new DOMException("The operation was aborted.", "AbortError"); + }); + + const { payload, error } = await resolved({ throwOnStatusError: true }); + + expect(error).toBeInstanceOf(DOMException); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: true }); + }); +}); + describe("buildDeployStatusReport", () => { const activeSnapshot = { appId: "app_1", diff --git a/packages/cli-core/src/commands/deploy/status.ts b/packages/cli-core/src/commands/deploy/status.ts index ceef5b7c1..f7eeba446 100644 --- a/packages/cli-core/src/commands/deploy/status.ts +++ b/packages/cli-core/src/commands/deploy/status.ts @@ -224,11 +224,11 @@ export async function resolveDeployState(ctx: DeployContext): Promise descriptor.provider); - const { productionConfig, deployStatus, live } = await loadProductionState( - ctx, - productionInstanceId, - domain.id, - options, + const completedProvidersIn = (config: Record): OAuthProvider[] => + oauthProviderDescriptors + .filter((descriptor) => hasProviderRequiredCredentials(config, descriptor)) + .map((descriptor) => descriptor.provider); + + const { productionConfig, deployStatus, live } = await withSpinner( + "Reading production configuration...", + async () => { + const configRead = Promise.resolve(fetchInstanceConfig(ctx.appId, productionInstanceId)).then( + (config) => { + recordOAuthObservation({ + oauthProviders, + completedOAuthProviders: completedProvidersIn(config), + }); + return config; + }, + ); + const statusRead = loadInitialDeployStatus(ctx.appId, domain.id, options).then((read) => { + if (read.live) + setTelemetryDomainComponents(deployComponentStatusFromDomainStatus(read.status)); + return read; + }); + const [productionConfig, { status: deployStatus, live }] = await settleBeforeRejecting([ + configRead, + statusRead, + ]); + return { productionConfig, deployStatus, live }; + }, ); - const completedOAuthProviders = oauthProviderDescriptors - .filter((descriptor) => hasProviderRequiredCredentials(productionConfig, descriptor)) - .map((descriptor) => descriptor.provider); + const completedOAuthProviders = completedProvidersIn(productionConfig); const pendingOAuthDescriptor = oauthProviderDescriptors.find( (descriptor) => !completedOAuthProviders.includes(descriptor.provider), ); @@ -318,6 +352,24 @@ export async function resolveLiveDeploySnapshot( }; } +/** + * `Promise.all`, except a rejection waits for the other promises to settle + * before it propagates. Same result and the same winning error — the first + * to fail — only the throw is delayed until a `.then` attached to a slower + * promise has run. Without this, whether that `.then` lands before or after + * the run finalizes its telemetry would be a race. + */ +async function settleBeforeRejecting( + promises: T, +): Promise<{ -readonly [P in keyof T]: Awaited }> { + try { + return await Promise.all(promises); + } catch (error) { + await Promise.allSettled(promises); + throw error; + } +} + function resolvePendingStep( pendingOAuthDescriptor: OAuthProviderDescriptor | undefined, domainComplete: boolean, @@ -348,25 +400,6 @@ export async function loadInitialDeployStatus( } } -export async function loadProductionState( - ctx: DeployContext, - productionInstanceId: string, - domainIdOrName: string, - options: SnapshotOptions = {}, -): Promise<{ - productionConfig: Record; - deployStatus: DomainStatusResponse; - live: boolean; -}> { - return withSpinner("Reading production configuration...", async () => { - const [productionConfig, { status: deployStatus, live }] = await Promise.all([ - fetchInstanceConfig(ctx.appId, productionInstanceId), - loadInitialDeployStatus(ctx.appId, domainIdOrName, options), - ]); - return { productionConfig, deployStatus, live }; - }); -} - export function pendingDomainStatus(): DomainStatusResponse { return { status: "incomplete", @@ -554,14 +587,19 @@ export function deployReportState( * {@link retractDeployStage}. An instance exists; nothing else is known. * - `startNewDeploy` on the create response: `domain_pending` or * `domain_provisioning`, from whether Clerk returned a domain. + * - `resolveLiveDeploySnapshot`, on every path that reads: `oauth` when the + * configuration read succeeds, the domain group when the domain-status + * read succeeds and is live — each at its own read. * - `reconcileExistingDeploy`: `domain_provisioning` when Clerk lists no - * domain, else the snapshot through `recordDeployObservation`. - * - `runOAuthSetup`, after each credential save: `oauth` alone, through - * {@link recordOAuthObservation}. + * domain, else the stage through `recordDeployObservation`. + * - `runOAuthSetup`, once every required credential is saved: `oauth: true`, + * through {@link recordOAuthObservation}. Never earlier — nothing has read + * the production configuration on a fresh deploy. * - `runDnsVerification`, once per poll: that poll, through `recordDeployPoll`. * - `finishDeploy`: the resolver over the OAuth facts and a verified domain. - * - `emitAgentDeployHandoff` and `deployStatus`: the state read through - * `recordDeployObservation`, then each poll through `recordDeployPoll`. + * - `emitAgentDeployHandoff` and `deployStatus`: the stage from the state + * read through `recordDeployObservation`, then each poll through + * `recordDeployPoll`. * * This entry takes a state the caller can vouch for without a snapshot — one * the CLI's own action established, or a poll's verdict. Anything derived @@ -580,15 +618,14 @@ export function recordOAuthObservation(oauth: OAuthSetupFacts): void { } /** - * Record everything a state read established. The four components come from - * two reads, so they are two observations: a snapshot exists only if the - * production-configuration read succeeded, so `oauth` is always recorded - * here, while the domain group and the stage need the domain-status read too - * and a substituted one records neither. This is the only place that stops a - * fallback being recorded as an observation: the callers that read through - * `resolveDeployState` never trip it today, because that read throws rather - * than substitutes, and the check is here so that stays true without every - * call site knowing about the option. + * Record the stage a state read established. The components are not recorded + * here: `resolveLiveDeploySnapshot` writes each the moment its own read + * succeeds, so a failure in the other read cannot discard it. The stage needs + * both reads, and a substituted domain read establishes none, so this is + * where that check lives — the callers that read through `resolveDeployState` + * never trip it today, because that read throws rather than substitutes, and + * the check is here so that stays true without every call site knowing about + * the option. */ export function recordDeployObservation(state: DeployState): void { if (state.kind !== "active") { @@ -596,17 +633,16 @@ export function recordDeployObservation(state: DeployState): void { return; } const { snapshot } = state; - recordOAuthObservation(snapshot); if (!snapshot.live) return; - setTelemetryDomainComponents(snapshot.componentStatus); recordDeployStage(resolveActiveReportState(snapshot, snapshot.domainComplete)); } /** * Record what one domain-status poll established: the three domain - * components and the stage. `oauth` is untouched — the poll did not read it, - * and the earlier observation stands. A poll is its own observation, so it - * records whatever the snapshot before it was. + * components and the stage. `oauth` is untouched because the poll did not + * observe it — not as an optimisation, but so a value the poll never learned + * cannot overwrite one an earlier read did. A poll is its own observation, so + * it records whatever the snapshot before it was. */ export function recordDeployPoll(oauth: OAuthSetupFacts, polled: DeployStatusOutcome): void { setTelemetryDomainComponents(polled.status); From 404580ac8c370ed12e540e6e8c140f55b05f123f Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 12:10:21 -0700 Subject: [PATCH 08/16] Record error codes for failures commands catch themselves --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 + .../cli-core/src/commands/api/index.test.ts | 100 ++++++++++++++++++ packages/cli-core/src/commands/api/index.ts | 3 + .../cli-core/src/commands/mcp/install.test.ts | 68 +++++++++++- packages/cli-core/src/commands/mcp/shared.ts | 5 +- .../src/commands/mcp/uninstall.test.ts | 21 +++- .../src/commands/users/create.test.ts | 55 +++++++++- .../interactive/instance-context.test.ts | 6 +- .../cli-core/src/commands/users/output.ts | 2 + .../cli-core/src/lib/bapi-command.test.ts | 29 ++++- packages/cli-core/src/lib/bapi-command.ts | 2 + packages/cli-core/src/lib/telemetry.test.ts | 75 ++++++++++++- packages/cli-core/src/lib/telemetry.ts | 66 +++++++++++- .../src/test/integration/telemetry.test.ts | 31 ++++++ 14 files changed, 454 insertions(+), 11 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 9006df813..c61ff8a47 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -5,3 +5,5 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Every `clerk deploy` and `clerk deploy status` event now also records the state the deploy was in when the run ended, so a run that stopped short says where, and which of DNS, SSL, email DNS and OAuth had been verified at that point — recorded only from a read that actually succeeded, so a failed status call is never reported as a failed check. Output and exit codes are unchanged. `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. + +`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (404), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. diff --git a/packages/cli-core/src/commands/api/index.test.ts b/packages/cli-core/src/commands/api/index.test.ts index 96d48a378..528df473f 100644 --- a/packages/cli-core/src/commands/api/index.test.ts +++ b/packages/cli-core/src/commands/api/index.test.ts @@ -10,6 +10,7 @@ import { configStubs, libPromptsStubs, stubFetch, + captureTelemetryPayload, } from "../../test/lib/stubs.ts"; let mockStoredToken: string | null = null; @@ -878,4 +879,103 @@ describe("api command", () => { Object.defineProperty(process, "platform", { value: originalPlatform, writable: true }); } }); + + // The API error is caught here to put the raw body on stdout, so the throw + // path never classifies it. What the event carries instead — per status, + // since telemetry has neither the status nor the endpoint. + describe("what telemetry records as the error code", () => { + const clerkBody = (code: string) => JSON.stringify({ errors: [{ code, message: "" }] }); + + function recordedFor(endpoint: string, options: Record = {}) { + return captureTelemetryPayload("api", () => runApi(endpoint, options)); + } + + test("a Clerk error code in the response body", async () => { + stubFetch(async () => new Response(clerkBody("resource_not_found"), { status: 404 })); + const { payload } = await recordedFor("/users/bad_id"); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("resource_not_found"); + }); + + test.each([ + [429, "api_rate_limited"], + [404, "api_not_found"], + [400, "api_client_error"], + [500, "api_error"], + ])("an uncoded %i is api-prefixed by status", async (status, expected) => { + stubFetch(async () => new Response("nope", { status })); + const { payload } = await recordedFor("/organization_role"); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe(expected); + }); + + // `--fapi` only decides whether there is a catalog to suggest searching; + // the status is the status. + test("--fapi makes no difference to an uncoded 404", async () => { + process.env.CLERK_PLATFORM_API_KEY = "ak_test_platform"; + const pk = `pk_test_${btoa("clerk.example.com$")}`; + stubFetch(async (input) => { + if (input.toString().includes("/v1/platform/applications/app_1")) { + return new Response( + JSON.stringify({ + application_id: "app_1", + instances: [ + { instance_id: "ins_dev", environment_type: "development", publishable_key: pk }, + ], + }), + { status: 200 }, + ); + } + return new Response("404 page not found", { status: 404 }); + }); + const { payload } = await recordedFor("/bogus", { + fapi: true, + app: "app_1", + instance: "dev", + }); + expect(payload.error_code).toBe("api_not_found"); + expect(captured.err).not.toContain("clerk api ls"); + }); + + // Not an ApiError, so the local catch rethrows and the throw path + // classifies it — unchanged, and pinned so the split cannot widen into it. + test("a rejected fetch is still a thrown unexpected_error", async () => { + stubFetch(async () => { + throw new Error("socket hang up"); + }); + const { payload, error } = await captureTelemetryPayload("api", () => runApi("/users"), { + captureError: true, + }); + expect(error).toBeInstanceOf(Error); + expect(payload.outcome).toBe("error"); + expect(payload.error_code).toBe("unexpected_error"); + }); + + test("a successful request is a success with no code", async () => { + const { payload } = await recordedFor("/users"); + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(0); + expect(payload.error_code).toBeNull(); + }); + + // The printed output is the reason the error is caught locally; recording + // it must not change a byte of it. + test("--include on an error still prints the headers and body, with the code on the event", async () => { + stubFetch( + async () => + new Response('{"error":"bad"}', { + status: 400, + headers: { "x-request-id": "req_err" }, + }), + ); + const { payload } = await recordedFor("/users", { include: true }); + expect(captured.err).toContain("HTTP 400"); + expect(captured.err).toContain("x-request-id: req_err"); + expect(captured.out).toContain('"error": "bad"'); + expect(payload.error_code).toBe("api_client_error"); + expect(payload.exit_code).toBe(1); + }); + }); }); diff --git a/packages/cli-core/src/commands/api/index.ts b/packages/cli-core/src/commands/api/index.ts index 34d35808f..3041f8a87 100644 --- a/packages/cli-core/src/commands/api/index.ts +++ b/packages/cli-core/src/commands/api/index.ts @@ -7,6 +7,7 @@ import { bapiRequest } from "../../lib/bapi.ts"; import { fapiRequest } from "../../lib/fapi.ts"; import { resolveFapiHost } from "./fapi.ts"; import { ApiError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { declareSoftExitError } from "../../lib/telemetry.ts"; import { validateJsonBody } from "../../lib/json-body.ts"; import { isHuman } from "../../mode.ts"; import { confirm } from "../../lib/prompts.ts"; @@ -154,6 +155,8 @@ export async function api( const scope = options.platform ? " --platform" : ""; log.info(`If the endpoint path was a guess, search with: clerk api ls ${scope}`); } + // Handled here, so telemetry never sees the throw it would classify. + declareSoftExitError(error); process.exitCode = 1; closeStatus = "failed"; return; diff --git a/packages/cli-core/src/commands/mcp/install.test.ts b/packages/cli-core/src/commands/mcp/install.test.ts index 2a7f2b4da..542aec00b 100644 --- a/packages/cli-core/src/commands/mcp/install.test.ts +++ b/packages/cli-core/src/commands/mcp/install.test.ts @@ -3,7 +3,7 @@ import { afterAll, afterEach, beforeEach, describe, expect, mock, test } from "b import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import * as realOs from "node:os"; import { join } from "node:path"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { captureTelemetryPayload, useCaptureLog } from "../../test/lib/stubs.ts"; const mockIsAgent = mock(); mock.module("../../mode.ts", () => ({ @@ -35,6 +35,7 @@ mock.module("./clients/cli-exec.ts", () => ({ afterAll(() => mock.restore()); const { mcpInstall } = await import("./install.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); // The URL the default env profile resolves to. const DEFAULT_URL = "https://mcp.clerk.com/mcp"; @@ -351,4 +352,69 @@ describe("mcp install", () => { }); expect(captured.err).toContain("unexpected flag"); }); + + // `--json` sets the exit code where human mode throws, so it used to record + // a bare error while human mode recorded the code. Both modes are driven + // here and must agree — that discrepancy is what GROW-1252 is about. + describe("what telemetry records as the error code", () => { + beforeEach(() => _setConfigDir(cwd)); + afterEach(() => _setConfigDir(undefined)); + + function recordedFor(options: Parameters[0]) { + return captureTelemetryPayload("mcp install", () => mcpInstall(options), { + captureError: !options?.json, + }); + } + + async function corruptCursorConfig() { + await mkdir(join(cwd, ".cursor"), { recursive: true }); + await writeFile(join(cwd, ".cursor", "mcp.json"), "{ not json"); + } + + test("a local config failure records the named CliError code in both modes", async () => { + await corruptCursorConfig(); + const json = await recordedFor({ client: ["cursor"], json: true }); + expect(json.payload.outcome).toBe("error"); + expect(json.payload.exit_code).toBe(1); + expect(json.payload.error_code).toBe("mcp_client_config_invalid"); + + const human = await recordedFor({ client: ["cursor"] }); + expect(human.error).toMatchObject({ code: "mcp_client_config_invalid" }); + expect(human.payload.error_code).toBe("mcp_client_config_invalid"); + expect(human.payload.exit_code).toBe(1); + }); + + test("a plain exception records unexpected_error in both modes", async () => { + mockRun.mockRejectedValue(new Error("spawn EAGAIN")); + const json = await recordedFor({ client: ["claude"], json: true }); + expect(json.payload.outcome).toBe("error"); + expect(json.payload.error_code).toBe("unexpected_error"); + + const human = await recordedFor({ client: ["claude"] }); + expect(human.error).toBeInstanceOf(Error); + expect(human.payload.error_code).toBe("unexpected_error"); + }); + + // The first client's error is the one reported, in either mode. + test("with several failed clients the first one's code is recorded", async () => { + await corruptCursorConfig(); + mockRun.mockRejectedValue(new Error("spawn EAGAIN")); + const { payload } = await recordedFor({ client: ["cursor", "claude"], json: true }); + expect(payload.error_code).toBe("mcp_client_config_invalid"); + }); + + test("a partial failure is still a success with no code", async () => { + await corruptCursorConfig(); + const { payload } = await recordedFor({ client: ["cursor", "windsurf"], json: true }); + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(0); + expect(payload.error_code).toBeNull(); + }); + + test("an install into every client is a success with no code", async () => { + const { payload } = await recordedFor({ client: ["cursor"], json: true }); + expect(payload.outcome).toBe("success"); + expect(payload.error_code).toBeNull(); + }); + }); }); diff --git a/packages/cli-core/src/commands/mcp/shared.ts b/packages/cli-core/src/commands/mcp/shared.ts index 7fc9102c3..42aa7e545 100644 --- a/packages/cli-core/src/commands/mcp/shared.ts +++ b/packages/cli-core/src/commands/mcp/shared.ts @@ -5,6 +5,7 @@ import { getMcpUrl } from "../../lib/environment.ts"; import { CliError, ERROR_CODE, errorMessage, throwUsageError } from "../../lib/errors.ts"; import { log } from "../../lib/log.ts"; +import { declareSoftExitError } from "../../lib/telemetry.ts"; import { isAgent } from "../../mode.ts"; import { CLIENT_ALIASES, CLIENT_IDS, CLIENTS, detectInstalledClients } from "./clients/registry.ts"; import { MCP_DOCS_URL } from "./clients/types.ts"; @@ -198,12 +199,14 @@ export async function settleClients( * Exit non-zero when every targeted client failed. In `--json` mode the * `{ results, failures }` envelope is already on stdout — exactly the case * where `failures` is most useful — so rethrowing would append a second - * (error) document and corrupt the stream; set the exit code instead. Human + * (error) document and corrupt the stream; set the exit code instead, and + * hand telemetry the same error so the two modes record the same code. Human * mode rethrows the first client's original error so the global handler * formats it with its code and docs URL. */ export function failWhenAllFailed(outcome: SettledClients, json: boolean): void { if (outcome.succeeded.length > 0 || outcome.firstError === undefined) return; if (!json) throw outcome.firstError; + declareSoftExitError(outcome.firstError); process.exitCode = 1; } diff --git a/packages/cli-core/src/commands/mcp/uninstall.test.ts b/packages/cli-core/src/commands/mcp/uninstall.test.ts index f6dd0228f..21318011d 100644 --- a/packages/cli-core/src/commands/mcp/uninstall.test.ts +++ b/packages/cli-core/src/commands/mcp/uninstall.test.ts @@ -3,7 +3,7 @@ import { afterAll, afterEach, beforeEach, describe, expect, mock, test } from "b import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import * as realOs from "node:os"; import { join } from "node:path"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { captureTelemetryPayload, useCaptureLog } from "../../test/lib/stubs.ts"; const mockIsAgent = mock(); mock.module("../../mode.ts", () => ({ @@ -34,6 +34,7 @@ afterAll(() => mock.restore()); const { mcpInstall } = await import("./install.ts"); const { mcpUninstall } = await import("./uninstall.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); const URL = "https://mcp.clerk.com/mcp"; const RUN_SHAPE = { command: "clerk", args: ["mcp", "run"] }; @@ -255,4 +256,22 @@ describe("mcp uninstall", () => { }; expect(cursorCfg.mcpServers?.clerk).toBeUndefined(); }); + + // Shares `failWhenAllFailed` with install, so the same fix reaches it: a + // total failure under `--json` records the code human mode would throw. + test("a total failure in JSON mode records the first client's error code", async () => { + _setConfigDir(cwd); + try { + await mkdir(join(cwd, ".cursor"), { recursive: true }); + await writeFile(join(cwd, ".cursor", "mcp.json"), "{ not json"); + const { payload } = await captureTelemetryPayload("mcp uninstall", () => + mcpUninstall({ client: ["cursor"] }), + ); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("mcp_client_config_invalid"); + } finally { + _setConfigDir(undefined); + } + }); }); diff --git a/packages/cli-core/src/commands/users/create.test.ts b/packages/cli-core/src/commands/users/create.test.ts index 67957b7b8..91fa05313 100644 --- a/packages/cli-core/src/commands/users/create.test.ts +++ b/packages/cli-core/src/commands/users/create.test.ts @@ -1,5 +1,8 @@ import { test, expect, describe, beforeEach, afterEach, mock, spyOn } from "bun:test"; -import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { captureTelemetryPayload, useCaptureLog } from "../../test/lib/stubs.ts"; import { BapiError, CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../lib/errors.ts"; const mockResolveBapiSecretKey = mock(); @@ -41,6 +44,7 @@ mock.module("../../lib/spinner.ts", () => ({ })); const { create } = await import("./create.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); describe("users create", () => { let logSpy: ReturnType; @@ -327,4 +331,53 @@ describe("users create", () => { expect(mockConfirm).not.toHaveBeenCalled(); expect(mockBapiRequest).not.toHaveBeenCalled(); }); + + // The BAPI error is caught to print Clerk's error body, so the throw path + // never classifies it; the event has to carry the code from the catch. + describe("what telemetry records as the error code", () => { + let configDir: string; + beforeEach(async () => { + configDir = await mkdtemp(join(tmpdir(), "clerk-users-create-telemetry-")); + _setConfigDir(configDir); + }); + afterEach(async () => { + _setConfigDir(undefined); + await rm(configDir, { recursive: true, force: true }); + }); + + const clerkBody = (code: string) => JSON.stringify({ errors: [{ code, message: "" }] }); + const input = { app: "app_123", email: "alice@example.com", yes: true }; + + function recordedFor(options: Parameters[0]) { + return captureTelemetryPayload("users create", () => runCreate(options)); + } + + test("a Clerk error code in the response body, in either output mode", async () => { + mockBapiRequest.mockRejectedValue( + BapiError.fromBody(422, clerkBody("form_param_missing"), new Headers()), + ); + const human = await recordedFor(input); + expect(human.payload.outcome).toBe("error"); + expect(human.payload.exit_code).toBe(1); + expect(human.payload.error_code).toBe("form_param_missing"); + + mockIsAgent.mockReturnValue(true); + const json = await recordedFor(input); + expect(json.payload.error_code).toBe("form_param_missing"); + }); + + test("an uncoded response is split by status like `clerk api`", async () => { + mockBapiRequest.mockRejectedValue(BapiError.fromBody(502, "bad gateway", new Headers())); + const { payload } = await recordedFor(input); + expect(payload.outcome).toBe("error"); + expect(payload.error_code).toBe("api_error"); + }); + + test("a created user is a success with no code", async () => { + const { payload } = await recordedFor(input); + expect(payload.outcome).toBe("success"); + expect(payload.exit_code).toBe(0); + expect(payload.error_code).toBeNull(); + }); + }); }); diff --git a/packages/cli-core/src/commands/users/interactive/instance-context.test.ts b/packages/cli-core/src/commands/users/interactive/instance-context.test.ts index ef94a615a..80cca41ca 100644 --- a/packages/cli-core/src/commands/users/interactive/instance-context.test.ts +++ b/packages/cli-core/src/commands/users/interactive/instance-context.test.ts @@ -1,6 +1,6 @@ import { test, expect, describe, beforeEach, afterEach, mock } from "bun:test"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; -import { stubFetch } from "../../../test/lib/stubs.ts"; +import { configStubs, stubFetch } from "../../../test/lib/stubs.ts"; const mockResolveAppContext = mock(); const mockResolveProfile = mock(); @@ -15,9 +15,7 @@ mock.module("../../../lib/listage.ts", () => ({ })); mock.module("../../../lib/config.ts", () => ({ - // fetch.ts (imported process-wide) reads these from config.ts. - getTelemetryDisabled: async () => false, - getTelemetryNoticeShown: async () => true, + ...configStubs, resolveAppContext: (...args: unknown[]) => mockResolveAppContext(...args), resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), resolveFetchedApplicationInstance: ( diff --git a/packages/cli-core/src/commands/users/output.ts b/packages/cli-core/src/commands/users/output.ts index b60cf9998..df0b53fb5 100644 --- a/packages/cli-core/src/commands/users/output.ts +++ b/packages/cli-core/src/commands/users/output.ts @@ -1,5 +1,6 @@ import { BapiError } from "../../lib/errors.ts"; import { log } from "../../lib/log.ts"; +import { declareSoftExitError } from "../../lib/telemetry.ts"; import { isAgent } from "../../mode.ts"; export type UsersOutputOptions = { @@ -74,6 +75,7 @@ export function handleUsersBapiError( log.error(`${context}: ${formatUsersErrorBody(error.body)}`); } + declareSoftExitError(error); process.exitCode = 1; return true; } diff --git a/packages/cli-core/src/lib/bapi-command.test.ts b/packages/cli-core/src/lib/bapi-command.test.ts index 6cc0e2800..b2c67e5ca 100644 --- a/packages/cli-core/src/lib/bapi-command.test.ts +++ b/packages/cli-core/src/lib/bapi-command.test.ts @@ -1,6 +1,9 @@ import { test, expect, describe, beforeEach, afterEach, spyOn } from "bun:test"; import { BapiError, CliError, ERROR_CODE } from "./errors.ts"; -import { useCaptureLog } from "../test/lib/stubs.ts"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { captureTelemetryPayload, useCaptureLog } from "../test/lib/stubs.ts"; const configModule = await import("./config.ts"); const plapiModule = await import("./plapi.ts"); @@ -73,6 +76,30 @@ describe("bapi-command", () => { expect(process.exitCode).toBe(1); }); + // Same catch-and-print shape as `users create`: the error never reaches the + // throw path, so this is where its code has to be handed to telemetry. + test("hands telemetry the code of the BAPI error it swallowed", async () => { + const configDir = await mkdtemp(join(tmpdir(), "clerk-bapi-command-telemetry-")); + configModule._setConfigDir(configDir); + try { + const { payload } = await captureTelemetryPayload("users ban", () => { + handleBapiError( + BapiError.fromBody( + 404, + JSON.stringify({ errors: [{ code: "resource_not_found", message: "" }] }), + new Headers(), + ), + ); + }); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("resource_not_found"); + } finally { + configModule._setConfigDir(undefined); + await rm(configDir, { recursive: true, force: true }); + } + }); + test("resolves secret key from explicit app and instance", async () => { fetchApplicationSpy.mockResolvedValue({ application_id: "app_123", diff --git a/packages/cli-core/src/lib/bapi-command.ts b/packages/cli-core/src/lib/bapi-command.ts index 0607325d6..234bed743 100644 --- a/packages/cli-core/src/lib/bapi-command.ts +++ b/packages/cli-core/src/lib/bapi-command.ts @@ -3,6 +3,7 @@ import { BapiError, CliError, ERROR_CODE, throwUsageError, withApiContext } from import { resolveKeylessTarget } from "./keyless-target.ts"; import { log } from "./log.ts"; import { fetchApplication, validateKeyPrefix } from "./plapi.ts"; +import { declareSoftExitError } from "./telemetry.ts"; export function normalizeBapiPath(path: string): string { let normalized = path; @@ -151,6 +152,7 @@ export function handleBapiError(error: unknown): boolean { log.data(error.body); } + declareSoftExitError(error); process.exitCode = 1; return true; } diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 3cde5cb04..7ae1ce735 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; import { + declareSoftExitError, declareSoftExitOutcome, finalizeAndSendTelemetry, getTelemetryStatus, @@ -18,7 +19,7 @@ import { type TelemetryCommand, type TelemetryResult, } from "./telemetry.ts"; -import { ApiError, CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "./errors.ts"; +import { ApiError, BapiError, CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "./errors.ts"; import { abortInFlight, beginInterrupt, _resetInterruptState } from "./signals.ts"; import { setLogLevel } from "./log.ts"; import { captureTelemetryPayload, fakeTelemetryCommand, useCaptureLog } from "../test/lib/stubs.ts"; @@ -643,6 +644,78 @@ describe("finalizeAndSendTelemetry", () => { test("declaring with no active context is a no-op", () => { expect(() => declareSoftExitOutcome("incomplete")).not.toThrow(); }); + + // The three commands that catch their own failure hand the error over + // here. Everything but an uncoded `ApiError` classifies exactly as a throw + // would, so `--json` and human mode of the same command record one code. + describe("a caught error carries the code a throw would", () => { + function codeFor(error: unknown): string | undefined { + startCommandTelemetry(fakeCommand()); + declareSoftExitError(error); + return telemetryResultForSoftExit(EXIT_CODE.GENERAL).errorCode; + } + + const clerkBody = (code: string) => JSON.stringify({ errors: [{ code, message: "" }] }); + + test("a Clerk error code in the body is recorded as-is, whatever the status", () => { + expect(codeFor(new ApiError(404, clerkBody("resource_not_found")))).toBe( + "resource_not_found", + ); + expect(codeFor(new ApiError(429, clerkBody("too_many_requests")))).toBe( + "too_many_requests", + ); + expect(codeFor(new BapiError(422, clerkBody("form_param_missing"), new Headers()))).toBe( + "form_param_missing", + ); + }); + + // No code in the body: the status is all that was observed, and each + // bucket claims exactly that. `too_many_requests` is deliberately not + // reused for the 429 — that code means Clerk itself said so. + test.each([ + [429, "api_rate_limited"], + [404, "api_not_found"], + [400, "api_client_error"], + [401, "api_client_error"], + [403, "api_client_error"], + [422, "api_client_error"], + [500, "api_error"], + [502, "api_error"], + [503, "api_error"], + ])("an uncoded %i records %s", (status, expected) => { + expect(codeFor(new ApiError(status, "not json"))).toBe(expected); + expect(codeFor(new ApiError(status, ""))).toBe(expected); + expect(codeFor(new ApiError(status, '{"error":"bad"}'))).toBe(expected); + }); + + test("a CliError keeps its named code", () => { + expect(codeFor(new CliError("boom", { code: ERROR_CODE.MCP_CLIENT_CONFIG_INVALID }))).toBe( + "mcp_client_config_invalid", + ); + expect(codeFor(new CliError("boom"))).toBe("cli_error"); + }); + + test("anything else is unexpected_error", () => { + expect(codeFor(new Error("EACCES"))).toBe("unexpected_error"); + expect(codeFor("just a string")).toBe("unexpected_error"); + }); + + test("the outcome is error, and only on a nonzero exit", async () => { + const failed = await sendAndCapturePayload( + () => declareSoftExitError(new ApiError(404, "")), + () => telemetryResultForSoftExit(EXIT_CODE.GENERAL), + ); + expect(failed.outcome).toBe("error"); + expect(failed.error_code).toBe("api_not_found"); + + const recovered = await sendAndCapturePayload( + () => declareSoftExitError(new ApiError(404, "")), + () => telemetryResultForSoftExit(EXIT_CODE.SUCCESS), + ); + expect(recovered.outcome).toBe("success"); + expect(recovered.error_code).toBeNull(); + }); + }); }); // The warehouse parses this payload by key. A rename splits a column in two diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index e3efa98aa..744631c93 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -365,7 +365,8 @@ export function setTelemetryOAuthComplete(complete: boolean): void { * so without this the soft-exit branch in `cli-program.ts` can only say * "nonzero, therefore error". That is wrong in both directions: `clerk deploy * status` exits 1 on a deploy that simply is not finished, and `clerk api` - * exits 1 holding an error code it never gets to record. + * exits 1 holding an error code a throw would have recorded (see + * {@link declareSoftExitError} for that side). * * Why this is a declaration and not a rule about exit codes: the exit code is * a per-command transport detail — 1 means "not done" from `deploy status` @@ -408,6 +409,69 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { }; } +/** + * Declare a failure the command caught and reported itself, carrying the code + * a throw would have. + * + * `clerk api`, `clerk users create` and `clerk mcp install --json` each catch + * their own error for a reason that stays as it is — the raw response body has + * to reach stdout for piping, or a second JSON document must not follow the + * first — and set the exit code instead. `telemetryResultForError` then never + * runs, and the code the error was holding is lost. This is the same + * classification, applied where the error is still in hand: a `CliError` + * keeps its named code, anything unrecognised is `unexpected_error`, so + * `mcp install --json` records what human mode records when it rethrows. + * + * The one difference from a throw: an `ApiError` with no parsed Clerk code is + * split by HTTP status rather than collapsed onto `api_error`. See + * {@link uncodedApiErrorCode} for why. Thrown `ApiError`s keep `api_error` + * because that code is on the warehouse's reviewed failure list as it is. + * + * Call it under the same condition that sets the exit code, and with the + * error the run means to report — the last-call-wins rule on + * {@link declareSoftExitOutcome} applies. Never hand it a `UserAbortError`: + * a declaration cannot express an abort, so it would be recorded as + * `unexpected_error`. A command that prompts inside a caught section must let + * the abort throw instead. (No caller can reach this today; the MCP client + * picker runs before any client is settled.) + */ +export function declareSoftExitError(error: unknown): void { + const code = + error instanceof ApiError + ? (error.code ?? uncodedApiErrorCode(error.status)) + : (telemetryResultForError(error).errorCode ?? "unexpected_error"); + declareSoftExitOutcome("error", code); +} + +/** + * An API response with no Clerk error code in its body has one HTTP status and + * no single meaning, so each code names exactly what was observed and nothing + * more. Telemetry carries no status and no endpoint, so this split is the only + * thing that makes the uncoded population measurable. + * + * - 429 → `api_rate_limited`, not the existing `too_many_requests`: that one + * arrives parsed from Clerk's error body, so it means Clerk itself said so. + * An uncoded 429 means no body said so — an empty body or an unexpected + * shape from Clerk parses the same as a proxy's answer, so the origin is + * unknown. Merging the two would erase the only distinction observable at + * the point of record. + * - 404 → `api_not_found`: commonly a URL path that does not exist, but the + * hint `clerk api` prints on this branch is a heuristic, so the code claims + * the status, not the cause. + * - other 4xx → `api_client_error`: 400, 401 and 403 collapsed. Cause and + * frequency unknown; the status cannot be recovered afterwards, so no + * finer mapping is promised. + * - anything else → `api_error`: a 5xx is a failed request whoever caused it, + * Clerk or a customer's proxy — the same ambiguity every thrown `ApiError` + * carries today. + */ +function uncodedApiErrorCode(status: number): string { + if (status === 429) return "api_rate_limited"; + if (status === 404) return "api_not_found"; + if (status >= 400 && status < 500) return "api_client_error"; + return "api_error"; +} + export function telemetryResultForError(error: unknown): TelemetryResult { if (error instanceof UserAbortError) { return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts index 975cdf9bf..b9d0c4b96 100644 --- a/packages/cli-core/src/test/integration/telemetry.test.ts +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -166,6 +166,37 @@ test("an unfinished `deploy status` is recorded as incomplete, not an error", as } }); +// `clerk api` catches the API error to print its body, so the code reaches +// the event only through the soft-exit declaration. M3's test above proves a +// declared outcome survives the real program; this proves a declared *code* +// does, and covers the status split end to end — the unit tests model the +// final step, this runs it. +test("a caught uncoded 404 from `clerk api` is recorded as api_not_found", async () => { + await markNoticeAlreadyShown(); + process.env.CLERK_TELEMETRY_URL = TELEMETRY_URL; + http.stub(async (url) => { + if (url.startsWith(TELEMETRY_URL)) return new Response("{}"); + return new Response("404 page not found", { status: 404 }); + }); + + try { + await clerk.raw("api", "/organization_role", "--secret-key", "sk_test_123"); + // Set by the command rather than thrown, so the harness's own result + // reports 0; the soft exit is on the process. + expect(process.exitCode).toBe(1); + + const bodies = telemetryEvents(); + expect(bodies).toHaveLength(1); + const event = bodies[0]!.events[0]!; + expect(event.payload.command).toBe("api"); + expect(event.payload.outcome).toBe("error"); + expect(event.payload.exit_code).toBe(1); + expect(event.payload.error_code).toBe("api_not_found"); + } finally { + process.exitCode = 0; + } +}); + // Drives the real program rather than the unit tests' capture helper, so it // pins two things only `runProgram` can: the command name Commander gives the // hidden default subcommand — the warehouse contract keys on `deploy run` — From 9e6f80636ea32c7a81257b07dacbbc09a1b8ddc9 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 14:18:20 -0700 Subject: [PATCH 09/16] Extract deploy telemetry recorders, trim comments --- .../cli-core/src/commands/deploy/index.ts | 24 +-- .../src/commands/deploy/report-state.ts | 33 ++++ .../cli-core/src/commands/deploy/state.ts | 11 +- .../src/commands/deploy/status-command.ts | 11 +- .../src/commands/deploy/status.test.ts | 3 +- .../cli-core/src/commands/deploy/status.ts | 147 +----------------- .../cli-core/src/commands/deploy/telemetry.ts | 71 +++++++++ packages/cli-core/src/lib/telemetry.ts | 25 +-- 8 files changed, 148 insertions(+), 177 deletions(-) create mode 100644 packages/cli-core/src/commands/deploy/report-state.ts create mode 100644 packages/cli-core/src/commands/deploy/telemetry.ts diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index eb0ee9b91..c3ea03684 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -65,23 +65,24 @@ import { import { buildDeployStatusReport, loadDevelopmentOAuthProviders, - recordDeployObservation, - recordDeployPoll, - recordDeployStage, - recordOAuthObservation, - resolveActiveReportState, resolveDeployContext, resolveDeployState, resolveLiveApplicationContext, resolveLiveDeploySnapshot, - retractDeployStage, waitForDeployStatus, type DeployProgressHandlers, type DeployStatusOutcome, type DiscoveredOAuthProviders, type LiveDeploySnapshot, - type OAuthSetupFacts, } from "./status.ts"; +import { clearTelemetryStage } from "../../lib/telemetry.ts"; +import { resolveActiveReportState, type OAuthSetupFacts } from "./report-state.ts"; +import { + recordDeployObservation, + recordDeployPoll, + recordDeployStage, + recordOAuthObservation, +} from "./telemetry.ts"; type DeployOptions = Record; @@ -185,10 +186,11 @@ async function startNewDeploy(ctx: DeployContext): Promise { const productionOrExists = await createProductionInstance(ctx, domain); if (productionOrExists === "exists") { - // `not_started` is now disproven, and nothing replaces it until the resume - // below reads the instance. If that read fails, or substitutes, the run - // ends with no stage rather than a false one. - retractDeployStage(); + // An observation that disproves the stage without establishing a new one: + // `not_started` is now false, and whether that instance has a domain, or + // how far it got, is unknown until the resume below reads it. If that read + // fails, or substitutes, the run ends with no stage rather than a false one. + clearTelemetryStage(); log.blank(); log.info( "A production instance already exists for this application. Resuming the existing deploy.", diff --git a/packages/cli-core/src/commands/deploy/report-state.ts b/packages/cli-core/src/commands/deploy/report-state.ts new file mode 100644 index 000000000..97f1a1507 --- /dev/null +++ b/packages/cli-core/src/commands/deploy/report-state.ts @@ -0,0 +1,33 @@ +import type { DeployOperationState } from "./state.ts"; +import type { DeployStatusState } from "./status.ts"; + +/** The states a deploy with a production domain can be in. */ +export type ActiveDeployStatusState = Extract< + DeployStatusState, + "domain_pending" | "oauth_pending" | "complete" +>; + +export type OAuthSetupFacts = Pick< + DeployOperationState, + "oauthProviders" | "completedOAuthProviders" +>; + +export function pendingOAuthProviders(oauth: OAuthSetupFacts): string[] { + return oauth.oauthProviders.filter( + (provider) => !oauth.completedOAuthProviders.includes(provider), + ); +} + +/** + * The one place the three active states are decided, for the report and for + * telemetry alike. `domainComplete` is the domain-status read's own verdict, + * not the three component booleans: all three can be verified while Clerk is + * still finalizing, and that is still `domain_pending`. + */ +export function resolveActiveReportState( + oauth: OAuthSetupFacts, + domainComplete: boolean, +): ActiveDeployStatusState { + if (!domainComplete) return "domain_pending"; + return pendingOAuthProviders(oauth).length === 0 ? "complete" : "oauth_pending"; +} diff --git a/packages/cli-core/src/commands/deploy/state.ts b/packages/cli-core/src/commands/deploy/state.ts index 8a78b9282..6ec30b3a4 100644 --- a/packages/cli-core/src/commands/deploy/state.ts +++ b/packages/cli-core/src/commands/deploy/state.ts @@ -59,13 +59,10 @@ const PAUSE_REASONS: Record< code: ErrorCode; exitCode: typeof EXIT_CODE.GENERAL | typeof EXIT_CODE.SIGINT; /** - * Whether the person stopped at a step. The CLI's own call — the warehouse - * does not enforce the pairing. Its payload contract test rejects a step - * outside `dns`/`oauth` wherever one appears, but only alarms on a step - * that stops arriving for `deploy_paused` and `deploy_cancelled` rows. So - * a fourth reason that invents a step value fails loudly, while one that - * simply needs adding to that alarm ships unmonitored until someone - * widens its eligibility list in `data-platform`. + * Whether the person stopped at a step. The CLI's own call: the warehouse + * validates the step value but only alarms on its absence for the codes + * it knows (as of data-platform#604), so a new reason that should carry a + * step also needs adding there. */ recordsPauseStep: boolean; } diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index fc6ba204b..0ce1a3db1 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -11,8 +11,6 @@ import { buildInterruptedDeployStatusReport, deployNextStep, loadProductionDomain, - recordDeployObservation, - recordDeployPoll, resolveDeployContext, resolveDeployState, triggerDeployStatusCheck, @@ -22,6 +20,7 @@ import { type DeployStatusOutcome, type DeployStatusReport, } from "./status.ts"; +import { recordDeployObservation, recordDeployPoll } from "./telemetry.ts"; import type { DeployContext } from "./state.ts"; type DeployStatusOptions = { @@ -89,9 +88,11 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise; - -export type OAuthSetupFacts = Pick< - DeployOperationState, - "oauthProviders" | "completedOAuthProviders" ->; - -function pendingOAuthProviders(oauth: OAuthSetupFacts): string[] { - return oauth.oauthProviders.filter( - (provider) => !oauth.completedOAuthProviders.includes(provider), - ); -} - -/** - * The one place the three active states are decided, for the report and for - * telemetry alike. `domainComplete` is the domain-status read's own verdict, - * not the three component booleans: all three can be verified while Clerk is - * still finalizing, and that is still `domain_pending`. - */ -export function resolveActiveReportState( - oauth: OAuthSetupFacts, - domainComplete: boolean, -): ActiveDeployStatusState { - if (!domainComplete) return "domain_pending"; - return pendingOAuthProviders(oauth).length === 0 ? "complete" : "oauth_pending"; -} - -/** - * The state a report for `state` carries. `outcome` is a wait's latest poll - * and overrides the snapshot's domain verdict when present. - */ -export function deployReportState( - state: DeployState, - outcome: DeployStatusOutcome | null, -): Exclude { - if (state.kind !== "active") return state.kind; - return resolveActiveReportState( - state.snapshot, - outcome ? outcome.verified : state.snapshot.domainComplete, - ); -} - -/** - * Record the deploy's state as telemetry's `stage`. Every write goes through - * this function, {@link recordDeployObservation} or {@link recordDeployPoll}, - * and every value is a report state — what `clerk deploy status` would print - * for this deploy at this moment — so the wizard, the agent handoff and the - * status command agree about the same deploy. - * - * One rule across every writer of `stage` and of the four `components`: last - * write wins, and nothing is written without an observation, so a run that - * ends before any state is known sends null. The writers, in the order a run - * can reach them: - * - * - `startNewDeploy` on entry: `not_started`. The create call has not run. - * - `startNewDeploy` when the create call finds an instance already exists: - * {@link retractDeployStage}. An instance exists; nothing else is known. - * - `startNewDeploy` on the create response: `domain_pending` or - * `domain_provisioning`, from whether Clerk returned a domain. - * - `resolveLiveDeploySnapshot`, on every path that reads: `oauth` when the - * configuration read succeeds, the domain group when the domain-status - * read succeeds and is live — each at its own read. - * - `reconcileExistingDeploy`: `domain_provisioning` when Clerk lists no - * domain, else the stage through `recordDeployObservation`. - * - `runOAuthSetup`, once every required credential is saved: `oauth: true`, - * through {@link recordOAuthObservation}. Never earlier — nothing has read - * the production configuration on a fresh deploy. - * - `runDnsVerification`, once per poll: that poll, through `recordDeployPoll`. - * - `finishDeploy`: the resolver over the OAuth facts and a verified domain. - * - `emitAgentDeployHandoff` and `deployStatus`: the stage from the state - * read through `recordDeployObservation`, then each poll through - * `recordDeployPoll`. - * - * This entry takes a state the caller can vouch for without a snapshot — one - * the CLI's own action established, or a poll's verdict. Anything derived - * from a snapshot goes through `recordDeployObservation`, which is where the - * substituted-read check lives. `interrupted` is not a state of the deploy: - * it means nothing was read, so whatever was last observed stays in place. - */ -export function recordDeployStage(state: DeployStatusState): void { - if (state === "interrupted") return; - setTelemetryStage(state); -} - -/** Record whether every required provider has production credentials. */ -export function recordOAuthObservation(oauth: OAuthSetupFacts): void { - setTelemetryOAuthComplete(pendingOAuthProviders(oauth).length === 0); -} - -/** - * Record the stage a state read established. The components are not recorded - * here: `resolveLiveDeploySnapshot` writes each the moment its own read - * succeeds, so a failure in the other read cannot discard it. The stage needs - * both reads, and a substituted domain read establishes none, so this is - * where that check lives — the callers that read through `resolveDeployState` - * never trip it today, because that read throws rather than substitutes, and - * the check is here so that stays true without every call site knowing about - * the option. - */ -export function recordDeployObservation(state: DeployState): void { - if (state.kind !== "active") { - recordDeployStage(state.kind); - return; - } - const { snapshot } = state; - if (!snapshot.live) return; - recordDeployStage(resolveActiveReportState(snapshot, snapshot.domainComplete)); -} - -/** - * Record what one domain-status poll established: the three domain - * components and the stage. `oauth` is untouched because the poll did not - * observe it — not as an optimisation, but so a value the poll never learned - * cannot overwrite one an earlier read did. A poll is its own observation, so - * it records whatever the snapshot before it was. - */ -export function recordDeployPoll(oauth: OAuthSetupFacts, polled: DeployStatusOutcome): void { - setTelemetryDomainComponents(polled.status); - recordDeployStage(resolveActiveReportState(oauth, polled.verified)); -} - -/** - * Forget the recorded stage. For the one case where an observation disproves - * the stage without establishing a new one: a fresh deploy's create call - * answering that an instance already exists. `not_started` is now false, and - * whether that instance has a domain, or how far it got, is unknown until the - * resume reads it — and the resume records normally when it does. - */ -export function retractDeployStage(): void { - clearTelemetryStage(); -} - /** * Classify what the reader should do next from the report's own fields, so * the human line rendered from a report and the agent sentence stored in it diff --git a/packages/cli-core/src/commands/deploy/telemetry.ts b/packages/cli-core/src/commands/deploy/telemetry.ts new file mode 100644 index 000000000..9e5ab420c --- /dev/null +++ b/packages/cli-core/src/commands/deploy/telemetry.ts @@ -0,0 +1,71 @@ +import { + setTelemetryDomainComponents, + setTelemetryOAuthComplete, + setTelemetryStage, +} from "../../lib/telemetry.ts"; +import { + pendingOAuthProviders, + resolveActiveReportState, + type OAuthSetupFacts, +} from "./report-state.ts"; +import type { DeployState, DeployStatusOutcome, DeployStatusState } from "./status.ts"; + +/** + * Record the deploy's state as telemetry's `stage`. Every write goes through + * this function, {@link recordDeployObservation} or {@link recordDeployPoll}, + * and every value is a report state — what `clerk deploy status` would print + * for this deploy at this moment — so the wizard, the agent handoff and the + * status command agree about the same deploy. + * + * One rule across every writer of `stage` and of the four `components`: last + * write wins, and nothing is written without an observation, so a run that + * ends before any state is known sends null. The endings are walked one by + * one in `index.test.ts` ("what telemetry records as the stage") and + * `status-command.test.ts`. + * + * This entry takes a state the caller can vouch for without a snapshot — one + * the CLI's own action established, or a poll's verdict. Anything derived + * from a snapshot goes through `recordDeployObservation`, which is where the + * substituted-read check lives. `interrupted` is not a state of the deploy, + * only of a report, so it cannot be recorded. + */ +export function recordDeployStage(state: Exclude): void { + setTelemetryStage(state); +} + +/** Record whether every required provider has production credentials. */ +export function recordOAuthObservation(oauth: OAuthSetupFacts): void { + setTelemetryOAuthComplete(pendingOAuthProviders(oauth).length === 0); +} + +/** + * Record the stage a state read established. The components are not recorded + * here: `resolveLiveDeploySnapshot` writes each the moment its own read + * succeeds, so a failure in the other read cannot discard it. The stage needs + * both reads, and a substituted domain read establishes none, so this is + * where that check lives — the callers that read through `resolveDeployState` + * never trip it today, because that read throws rather than substitutes, and + * the check is here so that stays true without every call site knowing about + * the option. + */ +export function recordDeployObservation(state: DeployState): void { + if (state.kind !== "active") { + recordDeployStage(state.kind); + return; + } + const { snapshot } = state; + if (!snapshot.live) return; + recordDeployStage(resolveActiveReportState(snapshot, snapshot.domainComplete)); +} + +/** + * Record what one domain-status poll established: the three domain + * components and the stage. `oauth` is untouched because the poll did not + * observe it — not as an optimisation, but so a value the poll never learned + * cannot overwrite one an earlier read did. A poll is its own observation, so + * it records whatever the snapshot before it was. + */ +export function recordDeployPoll(oauth: OAuthSetupFacts, polled: DeployStatusOutcome): void { + setTelemetryDomainComponents(polled.status); + recordDeployStage(resolveActiveReportState(oauth, polled.verified)); +} diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 744631c93..779d78aa9 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -54,11 +54,10 @@ export type TelemetryOutcome = "success" | "error" | "abort" | "incomplete"; * What a command may declare for itself on the soft-exit path. * * Deliberately narrower than {@link TelemetryOutcome}. `success` is excluded - * because declaring it on a run that then exits nonzero produces a row the - * warehouse reads as a success — its classifier tests `outcome = 'success'` - * ahead of every error rule and never reads `exit_code` — so the failure - * would leave the error series with nothing able to reconcile it. `abort` is - * excluded because it belongs to the interrupt path, which reports itself. + * because the warehouse classifies a row by `outcome` before it looks at + * anything else (as of data-platform#604), so declaring it on a run that then + * exits nonzero would file a failure as a success. `abort` is excluded because + * it belongs to the interrupt path, which reports itself. */ export type SoftExitOutcome = "incomplete" | "error"; @@ -121,7 +120,7 @@ export type TelemetryStage = // // Unlike the groups above, these are not control-flow positions: each is a // state of the deploy itself, as `resolveActiveReportState` in - // `commands/deploy/status.ts` would compute it at that moment. So the stage + // `commands/deploy/report-state.ts` would compute it at that moment. So the stage // a wizard run reports and the stage `clerk deploy status` reports a second // later agree about the same deploy. One value per run — the last state // observed, not every state the run passed through — and a run that ends @@ -296,9 +295,10 @@ export function setTelemetryStage(stage: TelemetryStage): void { /** * Forget the stage. For the one case where an observation disproves the - * stage last set without establishing a new one — `retractDeployStage` in - * `commands/deploy/status.ts` is the only caller. Not a general reset: a - * command that wants a different stage sets it. + * stage last set without establishing a new one — a fresh deploy's create + * call answering that an instance already exists, in `commands/deploy/index.ts`, + * is the only caller. Not a general reset: a command that wants a different + * stage sets it. */ export function clearTelemetryStage(): void { if (context) context.stage = null; @@ -560,6 +560,13 @@ async function buildAndSend( outcome: result.outcome, exit_code: result.exitCode, error_code: result.errorCode ?? null, + // `stage`, `pause_step` and `components` are deploy's and ride on every + // command's event as null. They sit at the top level because the + // warehouse staging model already reads these exact paths (as of + // data-platform#604), so nesting them under a per-command key now would + // cost a warehouse change for no visible gain. That is a cost call, not + // a shape to copy: a command that needs its own structured detail can + // still add a namespaced object, with a contract-test arm to match. stage: current.stage, pause_step: current.pauseStep, // Nested rather than four flat keys: it is one JSON path per component From b0ab02c948acb4283ed999e3cec640846d9f191d Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 14:33:15 -0700 Subject: [PATCH 10/16] Route every domain observation through one recorder --- .../cli-core/src/commands/deploy/status-command.ts | 6 +++--- packages/cli-core/src/commands/deploy/status.ts | 6 ++---- packages/cli-core/src/commands/deploy/telemetry.ts | 14 +++++++++++++- packages/cli-core/src/lib/telemetry.ts | 5 +++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/cli-core/src/commands/deploy/status-command.ts b/packages/cli-core/src/commands/deploy/status-command.ts index 0ce1a3db1..8cb522876 100644 --- a/packages/cli-core/src/commands/deploy/status-command.ts +++ b/packages/cli-core/src/commands/deploy/status-command.ts @@ -90,9 +90,9 @@ export async function deployStatus(options: DeployStatusOptions = {}): Promise { - if (read.live) - setTelemetryDomainComponents(deployComponentStatusFromDomainStatus(read.status)); + if (read.live) recordDomainObservation(deployComponentStatusFromDomainStatus(read.status)); return read; }); const [productionConfig, { status: deployStatus, live }] = await settleBeforeRejecting([ diff --git a/packages/cli-core/src/commands/deploy/telemetry.ts b/packages/cli-core/src/commands/deploy/telemetry.ts index 9e5ab420c..5bf80ef30 100644 --- a/packages/cli-core/src/commands/deploy/telemetry.ts +++ b/packages/cli-core/src/commands/deploy/telemetry.ts @@ -8,7 +8,10 @@ import { resolveActiveReportState, type OAuthSetupFacts, } from "./report-state.ts"; +// Type-only on purpose: `status.ts` imports the recorders at runtime, so a +// runtime import back would be an initialization-order cycle. import type { DeployState, DeployStatusOutcome, DeployStatusState } from "./status.ts"; +import type { DeployComponentStatus } from "./copy.ts"; /** * Record the deploy's state as telemetry's `stage`. Every write goes through @@ -38,6 +41,15 @@ export function recordOAuthObservation(oauth: OAuthSetupFacts): void { setTelemetryOAuthComplete(pendingOAuthProviders(oauth).length === 0); } +/** + * Record what one successful domain-status read said about DNS, SSL and email + * DNS. The initial read and every poll both come through here, so the two + * cannot drift apart. `oauth` is not this read's to write. + */ +export function recordDomainObservation(status: DeployComponentStatus): void { + setTelemetryDomainComponents(status); +} + /** * Record the stage a state read established. The components are not recorded * here: `resolveLiveDeploySnapshot` writes each the moment its own read @@ -66,6 +78,6 @@ export function recordDeployObservation(state: DeployState): void { * it records whatever the snapshot before it was. */ export function recordDeployPoll(oauth: OAuthSetupFacts, polled: DeployStatusOutcome): void { - setTelemetryDomainComponents(polled.status); + recordDomainObservation(polled.status); recordDeployStage(resolveActiveReportState(oauth, polled.verified)); } diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 779d78aa9..3cbba80b2 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -560,8 +560,9 @@ async function buildAndSend( outcome: result.outcome, exit_code: result.exitCode, error_code: result.errorCode ?? null, - // `stage`, `pause_step` and `components` are deploy's and ride on every - // command's event as null. They sit at the top level because the + // `stage` is shared (init, login and deploy each write their own group). + // `pause_step` and `components` are deploy's and ride on every other + // command's event as null members. They sit at the top level because the // warehouse staging model already reads these exact paths (as of // data-platform#604), so nesting them under a per-command key now would // cost a warehouse change for no visible gain. That is a cost call, not From 8e307744433298830b12a7ec2919288e48f866ad Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 16:31:11 -0700 Subject: [PATCH 11/16] Tell a typed 404 from one on a path the CLI built --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/api/index.test.ts | 15 ++++++++ packages/cli-core/src/commands/api/index.ts | 8 ++++- .../src/commands/api/interactive.test.ts | 35 ++++++++++++++++++- .../cli-core/src/commands/api/interactive.ts | 1 + .../src/commands/users/create.test.ts | 11 ++++++ packages/cli-core/src/lib/telemetry.test.ts | 21 +++++++++-- packages/cli-core/src/lib/telemetry.ts | 31 ++++++++++++---- 8 files changed, 111 insertions(+), 13 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index c61ff8a47..7860ebefb 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -6,4 +6,4 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than a `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. -`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (404), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. +`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. diff --git a/packages/cli-core/src/commands/api/index.test.ts b/packages/cli-core/src/commands/api/index.test.ts index 528df473f..61ec4f717 100644 --- a/packages/cli-core/src/commands/api/index.test.ts +++ b/packages/cli-core/src/commands/api/index.test.ts @@ -939,6 +939,21 @@ describe("api command", () => { expect(captured.err).not.toContain("clerk api ls"); }); + // The same bare 404 on a path the interactive builder chose from the + // CLI's own catalog is the CLI's failure, not a typo, and must not be + // filed with the typed ones. + test("an uncoded 404 on a catalog endpoint is cli_endpoint_not_found", async () => { + stubFetch(async () => new Response("404 page not found", { status: 404 })); + const { payload } = await recordedFor("/organization_role", { catalogEndpoint: true }); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("cli_endpoint_not_found"); + // A coded 404 names the resource, whoever wrote the path. + stubFetch(async () => new Response(clerkBody("resource_not_found"), { status: 404 })); + const coded = await recordedFor("/users/bad_id", { catalogEndpoint: true }); + expect(coded.payload.error_code).toBe("resource_not_found"); + }); + // Not an ApiError, so the local catch rethrows and the throw path // classifies it — unchanged, and pinned so the split cannot widen into it. test("a rejected fetch is still a thrown unexpected_error", async () => { diff --git a/packages/cli-core/src/commands/api/index.ts b/packages/cli-core/src/commands/api/index.ts index 3041f8a87..2b10696c2 100644 --- a/packages/cli-core/src/commands/api/index.ts +++ b/packages/cli-core/src/commands/api/index.ts @@ -27,6 +27,12 @@ export interface ApiOptions { fapi?: boolean; dryRun?: boolean; yes?: boolean; + /** + * Internal, not a flag: set by the interactive builder when the endpoint + * came from the CLI's own catalog rather than the command line, so a 404 + * on it is recorded as the CLI's failure and not the person's. + */ + catalogEndpoint?: boolean; } const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); @@ -156,7 +162,7 @@ export async function api( log.info(`If the endpoint path was a guess, search with: clerk api ls ${scope}`); } // Handled here, so telemetry never sees the throw it would classify. - declareSoftExitError(error); + declareSoftExitError(error, { userSuppliedPath: !options.catalogEndpoint }); process.exitCode = 1; closeStatus = "failed"; return; diff --git a/packages/cli-core/src/commands/api/interactive.test.ts b/packages/cli-core/src/commands/api/interactive.test.ts index 43c083128..aade2d7d7 100644 --- a/packages/cli-core/src/commands/api/interactive.test.ts +++ b/packages/cli-core/src/commands/api/interactive.test.ts @@ -2,7 +2,12 @@ import { test, expect, describe, beforeEach, afterEach, spyOn, mock } from "bun: import { mkdtemp, rm } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { useCaptureLog, listageStubs, stubFetch } from "../../test/lib/stubs.ts"; +import { + captureTelemetryPayload, + useCaptureLog, + listageStubs, + stubFetch, +} from "../../test/lib/stubs.ts"; let _mode = "human"; mock.module("../../mode.ts", () => ({ @@ -164,6 +169,34 @@ describe("apiInteractive", () => { expect(fetchCalls[0]!.method).toBe("GET"); }); + // The builder hands the catalog's path to the real handler; what telemetry + // records for a bare 404 on it is the seam this pins, end to end. + test("a bare 404 on a catalog endpoint is recorded as the CLI's failure", async () => { + setMode("human"); + selectResponses.push("Users"); + selectResponses.push({ + method: "GET", + path: "/users", + summary: "List all users", + tag: "Users", + operationId: "GetUserList", + pathParams: [], + hasRequestBody: false, + }); + confirmResponses.push(true); + stubFetch(async () => new Response("404 page not found", { status: 404 })); + const { _setConfigDir } = await import("../../lib/config.ts"); + _setConfigDir(tempDir); + try { + const { payload } = await captureTelemetryPayload("api", () => runApiInteractive({})); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("cli_endpoint_not_found"); + } finally { + _setConfigDir(undefined); + } + }); + test("prompts for path parameters", async () => { setMode("human"); selectResponses.push("Users"); diff --git a/packages/cli-core/src/commands/api/interactive.ts b/packages/cli-core/src/commands/api/interactive.ts index 0f42d7d4c..10a892a8e 100644 --- a/packages/cli-core/src/commands/api/interactive.ts +++ b/packages/cli-core/src/commands/api/interactive.ts @@ -105,5 +105,6 @@ export async function apiInteractive(options: ApiOptions): Promise { method: endpoint.method, data: body, yes: true, // skip double-confirmation + catalogEndpoint: true, }); } diff --git a/packages/cli-core/src/commands/users/create.test.ts b/packages/cli-core/src/commands/users/create.test.ts index 91fa05313..f56f3d5c2 100644 --- a/packages/cli-core/src/commands/users/create.test.ts +++ b/packages/cli-core/src/commands/users/create.test.ts @@ -373,6 +373,17 @@ describe("users create", () => { expect(payload.error_code).toBe("api_error"); }); + // The CLI built `/v1/users` itself, so a route the API does not serve is + // the CLI's failure — never `api_not_found`, which is reserved for a path + // the person typed. + test("an uncoded 404 is cli_endpoint_not_found, because the CLI wrote the path", async () => { + mockBapiRequest.mockRejectedValue( + BapiError.fromBody(404, "404 page not found", new Headers()), + ); + const { payload } = await recordedFor(input); + expect(payload.error_code).toBe("cli_endpoint_not_found"); + }); + test("a created user is a success with no code", async () => { const { payload } = await recordedFor(input); expect(payload.outcome).toBe("success"); diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 7ae1ce735..03f4d3585 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -671,10 +671,12 @@ describe("finalizeAndSendTelemetry", () => { // No code in the body: the status is all that was observed, and each // bucket claims exactly that. `too_many_requests` is deliberately not - // reused for the 429 — that code means Clerk itself said so. + // reused for the 429 — that code means Clerk itself said so. A 404 with + // nobody vouching for the path is the CLI's own route, so the default + // is the CLI's failure, not the person's. test.each([ [429, "api_rate_limited"], - [404, "api_not_found"], + [404, "cli_endpoint_not_found"], [400, "api_client_error"], [401, "api_client_error"], [403, "api_client_error"], @@ -688,6 +690,19 @@ describe("finalizeAndSendTelemetry", () => { expect(codeFor(new ApiError(status, '{"error":"bad"}'))).toBe(expected); }); + // Only a path the person typed can be the person's mistake. + test("an uncoded 404 on a path the person supplied is api_not_found", () => { + startCommandTelemetry(fakeCommand()); + declareSoftExitError(new ApiError(404, "404 page not found"), { userSuppliedPath: true }); + expect(telemetryResultForSoftExit(EXIT_CODE.GENERAL).errorCode).toBe("api_not_found"); + + startCommandTelemetry(fakeCommand()); + declareSoftExitError(new ApiError(404, clerkBody("resource_not_found")), { + userSuppliedPath: true, + }); + expect(telemetryResultForSoftExit(EXIT_CODE.GENERAL).errorCode).toBe("resource_not_found"); + }); + test("a CliError keeps its named code", () => { expect(codeFor(new CliError("boom", { code: ERROR_CODE.MCP_CLIENT_CONFIG_INVALID }))).toBe( "mcp_client_config_invalid", @@ -702,7 +717,7 @@ describe("finalizeAndSendTelemetry", () => { test("the outcome is error, and only on a nonzero exit", async () => { const failed = await sendAndCapturePayload( - () => declareSoftExitError(new ApiError(404, "")), + () => declareSoftExitError(new ApiError(404, ""), { userSuppliedPath: true }), () => telemetryResultForSoftExit(EXIT_CODE.GENERAL), ); expect(failed.outcome).toBe("error"); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 3cbba80b2..aed634b13 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -427,6 +427,13 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { * {@link uncodedApiErrorCode} for why. Thrown `ApiError`s keep `api_error` * because that code is on the warehouse's reviewed failure list as it is. * + * `userSuppliedPath` says who wrote the request path, which only the call + * site knows and which decides what an uncoded 404 means: a person's typo, or + * the CLI asking for a route the API does not serve. It defaults to the CLI, + * because every caller but one builds its own paths; `clerk api` passes true + * for a path typed on the command line and false for one its interactive + * builder chose from the endpoint catalog. + * * Call it under the same condition that sets the exit code, and with the * error the run means to report — the last-call-wins rule on * {@link declareSoftExitOutcome} applies. Never hand it a `UserAbortError`: @@ -435,10 +442,13 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { * the abort throw instead. (No caller can reach this today; the MCP client * picker runs before any client is settled.) */ -export function declareSoftExitError(error: unknown): void { +export function declareSoftExitError( + error: unknown, + options: { userSuppliedPath?: boolean } = {}, +): void { const code = error instanceof ApiError - ? (error.code ?? uncodedApiErrorCode(error.status)) + ? (error.code ?? uncodedApiErrorCode(error.status, options.userSuppliedPath === true)) : (telemetryResultForError(error).errorCode ?? "unexpected_error"); declareSoftExitOutcome("error", code); } @@ -455,9 +465,16 @@ export function declareSoftExitError(error: unknown): void { * shape from Clerk parses the same as a proxy's answer, so the origin is * unknown. Merging the two would erase the only distinction observable at * the point of record. - * - 404 → `api_not_found`: commonly a URL path that does not exist, but the - * hint `clerk api` prints on this branch is a heuristic, so the code claims - * the status, not the cause. + * - 404 with a path the person typed → `api_not_found`: the path did not + * reach a Clerk route, and the person chose it. The hint `clerk api` + * prints on this branch is a heuristic, so the code claims the status and + * who wrote the path, not the cause. + * - 404 with a path the CLI built → `cli_endpoint_not_found`: the CLI asked + * for a route the API does not serve, from a stale endpoint catalog or a + * hardcoded path, so this is the CLI's failure and the warehouse counts it + * as one. Kept apart from `api_not_found` because the same status means + * opposite things depending on who wrote the path, and the row cannot say + * which afterwards. * - other 4xx → `api_client_error`: 400, 401 and 403 collapsed. Cause and * frequency unknown; the status cannot be recovered afterwards, so no * finer mapping is promised. @@ -465,9 +482,9 @@ export function declareSoftExitError(error: unknown): void { * Clerk or a customer's proxy — the same ambiguity every thrown `ApiError` * carries today. */ -function uncodedApiErrorCode(status: number): string { +function uncodedApiErrorCode(status: number, userSuppliedPath: boolean): string { if (status === 429) return "api_rate_limited"; - if (status === 404) return "api_not_found"; + if (status === 404) return userSuppliedPath ? "api_not_found" : "cli_endpoint_not_found"; if (status >= 400 && status < 500) return "api_client_error"; return "api_error"; } From a0a769e67ba23fec19f2bbf18c262822ec063179 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 17:09:26 -0700 Subject: [PATCH 12/16] Require the path's author at every caught 404 --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/api/index.test.ts | 4 +- packages/cli-core/src/commands/api/index.ts | 11 +++--- .../src/commands/api/interactive.test.ts | 39 ++++++++++++++----- .../cli-core/src/commands/api/interactive.ts | 6 ++- packages/cli-core/src/commands/mcp/shared.ts | 2 +- .../cli-core/src/commands/users/output.ts | 2 +- packages/cli-core/src/lib/bapi-command.ts | 2 +- packages/cli-core/src/lib/telemetry.test.ts | 8 ++-- packages/cli-core/src/lib/telemetry.ts | 26 ++++++------- 10 files changed, 62 insertions(+), 40 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 7860ebefb..4d04f3f93 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -6,4 +6,4 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than a `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. -`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. +`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. The interactive builder now URL-escapes the path parameters you type, so a value containing `/`, `?` or `#` can no longer change the route. Otherwise nothing printed changes and exit codes are unchanged. diff --git a/packages/cli-core/src/commands/api/index.test.ts b/packages/cli-core/src/commands/api/index.test.ts index 61ec4f717..95f408bdf 100644 --- a/packages/cli-core/src/commands/api/index.test.ts +++ b/packages/cli-core/src/commands/api/index.test.ts @@ -944,13 +944,13 @@ describe("api command", () => { // filed with the typed ones. test("an uncoded 404 on a catalog endpoint is cli_endpoint_not_found", async () => { stubFetch(async () => new Response("404 page not found", { status: 404 })); - const { payload } = await recordedFor("/organization_role", { catalogEndpoint: true }); + const { payload } = await recordedFor("/organization_role", { userSuppliedPath: false }); expect(payload.outcome).toBe("error"); expect(payload.exit_code).toBe(1); expect(payload.error_code).toBe("cli_endpoint_not_found"); // A coded 404 names the resource, whoever wrote the path. stubFetch(async () => new Response(clerkBody("resource_not_found"), { status: 404 })); - const coded = await recordedFor("/users/bad_id", { catalogEndpoint: true }); + const coded = await recordedFor("/users/bad_id", { userSuppliedPath: false }); expect(coded.payload.error_code).toBe("resource_not_found"); }); diff --git a/packages/cli-core/src/commands/api/index.ts b/packages/cli-core/src/commands/api/index.ts index 2b10696c2..3f5a54150 100644 --- a/packages/cli-core/src/commands/api/index.ts +++ b/packages/cli-core/src/commands/api/index.ts @@ -28,11 +28,12 @@ export interface ApiOptions { dryRun?: boolean; yes?: boolean; /** - * Internal, not a flag: set by the interactive builder when the endpoint - * came from the CLI's own catalog rather than the command line, so a 404 - * on it is recorded as the CLI's failure and not the person's. + * Internal, not a flag. Who wrote the request path, for telemetry's 404 + * classification. Unset means the command line, so the person. The + * interactive builder passes false for an endpoint it chose from the CLI's + * own catalog, so a 404 on it is recorded as the CLI's failure. */ - catalogEndpoint?: boolean; + userSuppliedPath?: boolean; } const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); @@ -162,7 +163,7 @@ export async function api( log.info(`If the endpoint path was a guess, search with: clerk api ls ${scope}`); } // Handled here, so telemetry never sees the throw it would classify. - declareSoftExitError(error, { userSuppliedPath: !options.catalogEndpoint }); + declareSoftExitError(error, { userSuppliedPath: options.userSuppliedPath ?? true }); process.exitCode = 1; closeStatus = "failed"; return; diff --git a/packages/cli-core/src/commands/api/interactive.test.ts b/packages/cli-core/src/commands/api/interactive.test.ts index aade2d7d7..79365203b 100644 --- a/packages/cli-core/src/commands/api/interactive.test.ts +++ b/packages/cli-core/src/commands/api/interactive.test.ts @@ -20,6 +20,7 @@ mock.module("../../mode.ts", () => ({ })); const { parseSpec, _setCacheDir } = (await import("./catalog.ts")) as any; +const { _setConfigDir } = await import("../../lib/config.ts"); const { setMode } = (await import("../../mode.ts")) as any; const MINIMAL_SPEC = ` @@ -86,6 +87,7 @@ describe("apiInteractive", () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), "clerk-interactive-test-")); _setCacheDir(tempDir); + _setConfigDir(tempDir); // Pre-populate fresh cache const cached = parseSpec(MINIMAL_SPEC); @@ -120,6 +122,7 @@ describe("apiInteractive", () => { afterEach(async () => { _setCacheDir(undefined); + _setConfigDir(undefined); process.env = { ...originalEnv }; globalThis.fetch = originalFetch; Object.defineProperty(process.stdin, "isTTY", { @@ -185,16 +188,32 @@ describe("apiInteractive", () => { }); confirmResponses.push(true); stubFetch(async () => new Response("404 page not found", { status: 404 })); - const { _setConfigDir } = await import("../../lib/config.ts"); - _setConfigDir(tempDir); - try { - const { payload } = await captureTelemetryPayload("api", () => runApiInteractive({})); - expect(payload.outcome).toBe("error"); - expect(payload.exit_code).toBe(1); - expect(payload.error_code).toBe("cli_endpoint_not_found"); - } finally { - _setConfigDir(undefined); - } + const { payload } = await captureTelemetryPayload("api", () => runApiInteractive({})); + expect(payload.outcome).toBe("error"); + expect(payload.exit_code).toBe(1); + expect(payload.error_code).toBe("cli_endpoint_not_found"); + }); + + // A typed parameter is encoded, so it cannot turn `/users/{user_id}` into a + // different route; the request the API sees is still the catalog's. + test("a typed path parameter is URL-encoded", async () => { + setMode("human"); + selectResponses.push("Users"); + selectResponses.push({ + method: "GET", + path: "/users/{user_id}", + summary: "Retrieve a user", + tag: "Users", + operationId: "GetUser", + pathParams: [{ name: "user_id", description: "" }], + hasRequestBody: false, + }); + inputResponses.push("abc/def ghi"); + confirmResponses.push(true); + + await runApiInteractive({}); + + expect(fetchCalls[0]!.url).toContain("/v1/users/abc%2Fdef%20ghi"); }); test("prompts for path parameters", async () => { diff --git a/packages/cli-core/src/commands/api/interactive.ts b/packages/cli-core/src/commands/api/interactive.ts index 10a892a8e..1d9c69d4a 100644 --- a/packages/cli-core/src/commands/api/interactive.ts +++ b/packages/cli-core/src/commands/api/interactive.ts @@ -54,7 +54,9 @@ export async function apiInteractive(options: ApiOptions): Promise { message: param.description ? `${param.name} (${param.description}):` : `${param.name}:`, validate: (v) => (v?.trim() ? undefined : `${param.name} is required`), }); - resolvedPath = resolvedPath.replace(`{${param.name}}`, value.trim()); + // Encoded so a typed value cannot change the route: a 404 on the result + // is then the catalog's endpoint failing, never the person's input. + resolvedPath = resolvedPath.replace(`{${param.name}}`, encodeURIComponent(value.trim())); } // 5. Request body (if applicable) @@ -105,6 +107,6 @@ export async function apiInteractive(options: ApiOptions): Promise { method: endpoint.method, data: body, yes: true, // skip double-confirmation - catalogEndpoint: true, + userSuppliedPath: false, }); } diff --git a/packages/cli-core/src/commands/mcp/shared.ts b/packages/cli-core/src/commands/mcp/shared.ts index 42aa7e545..202c696e8 100644 --- a/packages/cli-core/src/commands/mcp/shared.ts +++ b/packages/cli-core/src/commands/mcp/shared.ts @@ -207,6 +207,6 @@ export async function settleClients( export function failWhenAllFailed(outcome: SettledClients, json: boolean): void { if (outcome.succeeded.length > 0 || outcome.firstError === undefined) return; if (!json) throw outcome.firstError; - declareSoftExitError(outcome.firstError); + declareSoftExitError(outcome.firstError, { userSuppliedPath: false }); process.exitCode = 1; } diff --git a/packages/cli-core/src/commands/users/output.ts b/packages/cli-core/src/commands/users/output.ts index df0b53fb5..116723738 100644 --- a/packages/cli-core/src/commands/users/output.ts +++ b/packages/cli-core/src/commands/users/output.ts @@ -75,7 +75,7 @@ export function handleUsersBapiError( log.error(`${context}: ${formatUsersErrorBody(error.body)}`); } - declareSoftExitError(error); + declareSoftExitError(error, { userSuppliedPath: false }); process.exitCode = 1; return true; } diff --git a/packages/cli-core/src/lib/bapi-command.ts b/packages/cli-core/src/lib/bapi-command.ts index 234bed743..5581358af 100644 --- a/packages/cli-core/src/lib/bapi-command.ts +++ b/packages/cli-core/src/lib/bapi-command.ts @@ -152,7 +152,7 @@ export function handleBapiError(error: unknown): boolean { log.data(error.body); } - declareSoftExitError(error); + declareSoftExitError(error, { userSuppliedPath: false }); process.exitCode = 1; return true; } diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 03f4d3585..4f1e884e8 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -651,7 +651,7 @@ describe("finalizeAndSendTelemetry", () => { describe("a caught error carries the code a throw would", () => { function codeFor(error: unknown): string | undefined { startCommandTelemetry(fakeCommand()); - declareSoftExitError(error); + declareSoftExitError(error, { userSuppliedPath: false }); return telemetryResultForSoftExit(EXIT_CODE.GENERAL).errorCode; } @@ -717,14 +717,14 @@ describe("finalizeAndSendTelemetry", () => { test("the outcome is error, and only on a nonzero exit", async () => { const failed = await sendAndCapturePayload( - () => declareSoftExitError(new ApiError(404, ""), { userSuppliedPath: true }), + () => declareSoftExitError(new ApiError(404, ""), { userSuppliedPath: false }), () => telemetryResultForSoftExit(EXIT_CODE.GENERAL), ); expect(failed.outcome).toBe("error"); - expect(failed.error_code).toBe("api_not_found"); + expect(failed.error_code).toBe("cli_endpoint_not_found"); const recovered = await sendAndCapturePayload( - () => declareSoftExitError(new ApiError(404, "")), + () => declareSoftExitError(new ApiError(404, ""), { userSuppliedPath: false }), () => telemetryResultForSoftExit(EXIT_CODE.SUCCESS), ); expect(recovered.outcome).toBe("success"); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index aed634b13..117a83e6b 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -429,10 +429,12 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { * * `userSuppliedPath` says who wrote the request path, which only the call * site knows and which decides what an uncoded 404 means: a person's typo, or - * the CLI asking for a route the API does not serve. It defaults to the CLI, - * because every caller but one builds its own paths; `clerk api` passes true + * the CLI asking for a route the API does not serve. It is required rather + * than defaulted so a new call site cannot mis-file a 404 by omission: the + * BAPI commands build their own paths and pass false; `clerk api` passes true * for a path typed on the command line and false for one its interactive - * builder chose from the endpoint catalog. + * builder chose from the endpoint catalog, whose path parameters are + * URL-encoded so a typed value cannot change the route. * * Call it under the same condition that sets the exit code, and with the * error the run means to report — the last-call-wins rule on @@ -442,13 +444,10 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { * the abort throw instead. (No caller can reach this today; the MCP client * picker runs before any client is settled.) */ -export function declareSoftExitError( - error: unknown, - options: { userSuppliedPath?: boolean } = {}, -): void { +export function declareSoftExitError(error: unknown, options: { userSuppliedPath: boolean }): void { const code = error instanceof ApiError - ? (error.code ?? uncodedApiErrorCode(error.status, options.userSuppliedPath === true)) + ? (error.code ?? uncodedApiErrorCode(error.status, options.userSuppliedPath)) : (telemetryResultForError(error).errorCode ?? "unexpected_error"); declareSoftExitOutcome("error", code); } @@ -470,11 +469,12 @@ export function declareSoftExitError( * prints on this branch is a heuristic, so the code claims the status and * who wrote the path, not the cause. * - 404 with a path the CLI built → `cli_endpoint_not_found`: the CLI asked - * for a route the API does not serve, from a stale endpoint catalog or a - * hardcoded path, so this is the CLI's failure and the warehouse counts it - * as one. Kept apart from `api_not_found` because the same status means - * opposite things depending on who wrote the path, and the row cannot say - * which afterwards. + * for a route and nothing served it — a stale endpoint catalog, a hardcoded + * path the API dropped, or something in front of the API answering for it + * (`CLERK_BACKEND_API_URL` is overridable), the same ambiguity the 5xx + * bullet carries. The warehouse counts it as a failure. Kept apart from + * `api_not_found` because the same status means opposite things depending + * on who wrote the path, and the row cannot say which afterwards. * - other 4xx → `api_client_error`: 400, 401 and 403 collapsed. Cause and * frequency unknown; the status cannot be recovered afterwards, so no * finer mapping is promised. From d64a638b6cb7ec80b2220019254afadeabac0543 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 17:18:32 -0700 Subject: [PATCH 13/16] Keep the builder telemetry-only --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- packages/cli-core/src/commands/api/index.ts | 8 ++++--- .../src/commands/api/interactive.test.ts | 22 ------------------- .../cli-core/src/commands/api/interactive.ts | 7 +++--- packages/cli-core/src/commands/mcp/shared.ts | 3 +++ packages/cli-core/src/lib/telemetry.ts | 3 +-- 6 files changed, 14 insertions(+), 31 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 4d04f3f93..7860ebefb 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -6,4 +6,4 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than a `clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. -`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. The interactive builder now URL-escapes the path parameters you type, so a value containing `/`, `?` or `#` can no longer change the route. Otherwise nothing printed changes and exit codes are unchanged. +`clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. diff --git a/packages/cli-core/src/commands/api/index.ts b/packages/cli-core/src/commands/api/index.ts index 3f5a54150..1bd1aada2 100644 --- a/packages/cli-core/src/commands/api/index.ts +++ b/packages/cli-core/src/commands/api/index.ts @@ -29,9 +29,11 @@ export interface ApiOptions { yes?: boolean; /** * Internal, not a flag. Who wrote the request path, for telemetry's 404 - * classification. Unset means the command line, so the person. The - * interactive builder passes false for an endpoint it chose from the CLI's - * own catalog, so a 404 on it is recorded as the CLI's failure. + * classification. Unset means the command line, so the person — safe only + * while Commander's registration and the interactive builder are the only + * callers. A caller that builds its own path must pass false, as the + * builder does for an endpoint chosen from the CLI's own catalog, so a 404 + * on it is recorded as the CLI's failure. */ userSuppliedPath?: boolean; } diff --git a/packages/cli-core/src/commands/api/interactive.test.ts b/packages/cli-core/src/commands/api/interactive.test.ts index 79365203b..ff54608e6 100644 --- a/packages/cli-core/src/commands/api/interactive.test.ts +++ b/packages/cli-core/src/commands/api/interactive.test.ts @@ -194,28 +194,6 @@ describe("apiInteractive", () => { expect(payload.error_code).toBe("cli_endpoint_not_found"); }); - // A typed parameter is encoded, so it cannot turn `/users/{user_id}` into a - // different route; the request the API sees is still the catalog's. - test("a typed path parameter is URL-encoded", async () => { - setMode("human"); - selectResponses.push("Users"); - selectResponses.push({ - method: "GET", - path: "/users/{user_id}", - summary: "Retrieve a user", - tag: "Users", - operationId: "GetUser", - pathParams: [{ name: "user_id", description: "" }], - hasRequestBody: false, - }); - inputResponses.push("abc/def ghi"); - confirmResponses.push(true); - - await runApiInteractive({}); - - expect(fetchCalls[0]!.url).toContain("/v1/users/abc%2Fdef%20ghi"); - }); - test("prompts for path parameters", async () => { setMode("human"); selectResponses.push("Users"); diff --git a/packages/cli-core/src/commands/api/interactive.ts b/packages/cli-core/src/commands/api/interactive.ts index 1d9c69d4a..40e14c9c4 100644 --- a/packages/cli-core/src/commands/api/interactive.ts +++ b/packages/cli-core/src/commands/api/interactive.ts @@ -54,9 +54,7 @@ export async function apiInteractive(options: ApiOptions): Promise { message: param.description ? `${param.name} (${param.description}):` : `${param.name}:`, validate: (v) => (v?.trim() ? undefined : `${param.name} is required`), }); - // Encoded so a typed value cannot change the route: a 404 on the result - // is then the catalog's endpoint failing, never the person's input. - resolvedPath = resolvedPath.replace(`{${param.name}}`, encodeURIComponent(value.trim())); + resolvedPath = resolvedPath.replace(`{${param.name}}`, value.trim()); } // 5. Request body (if applicable) @@ -107,6 +105,9 @@ export async function apiInteractive(options: ApiOptions): Promise { method: endpoint.method, data: body, yes: true, // skip double-confirmation + // The endpoint is the catalog's, so a bare 404 is recorded as the CLI's. + // A typed parameter goes into the route as typed, so a malformed value + // (a slash, a dot segment) can be misfiled the same way; accepted as rare. userSuppliedPath: false, }); } diff --git a/packages/cli-core/src/commands/mcp/shared.ts b/packages/cli-core/src/commands/mcp/shared.ts index 202c696e8..498748b1d 100644 --- a/packages/cli-core/src/commands/mcp/shared.ts +++ b/packages/cli-core/src/commands/mcp/shared.ts @@ -207,6 +207,9 @@ export async function settleClients( export function failWhenAllFailed(outcome: SettledClients, json: boolean): void { if (outcome.succeeded.length > 0 || outcome.firstError === undefined) return; if (!json) throw outcome.firstError; + // No request path here: client failures are local `CliError`s, so the + // answer is never read. It is `false` so that if a client ever surfaces an + // `ApiError`, its route is on record as the CLI's. declareSoftExitError(outcome.firstError, { userSuppliedPath: false }); process.exitCode = 1; } diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 117a83e6b..abf7feaa1 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -433,8 +433,7 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { * than defaulted so a new call site cannot mis-file a 404 by omission: the * BAPI commands build their own paths and pass false; `clerk api` passes true * for a path typed on the command line and false for one its interactive - * builder chose from the endpoint catalog, whose path parameters are - * URL-encoded so a typed value cannot change the route. + * builder chose from the endpoint catalog. * * Call it under the same condition that sets the exit code, and with the * error the run means to report — the last-call-wins rule on From fe22214a7b300ccc749ae90fd5a5f0ff35a19252 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 17:35:17 -0700 Subject: [PATCH 14/16] Fail fast on a deploy read and fix the review leftovers --- .../cli-core/src/commands/deploy/README.md | 2 +- .../src/commands/deploy/index.test.ts | 16 ++++++++++++ .../src/commands/deploy/status.test.ts | 7 ++++-- .../cli-core/src/commands/deploy/status.ts | 25 +++++-------------- packages/cli-core/src/lib/telemetry.test.ts | 4 +-- packages/cli-core/src/lib/telemetry.ts | 4 +-- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 43d6952c9..2eb0da6af 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -77,7 +77,7 @@ Four ways a run ends with the deploy unfinished and nothing broken. All four exi | Every DNS component verified, Clerk still provisioning | `error` | `deploy_finalizing` | null | 1 | | The user chose "skip" at DNS verification | `success` | null | null | 0 | -`pause_step` is null on the finalizing row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. The DNS skip is a finished command, not a pause — the wizard prints its summary and exits 0 — so it carries no code, and it is the row to remember when the `paused` class contains no DNS traffic. A Ctrl-C _before_ the production instance exists is not any of these either — there is no state to preserve, so it stays a plain `abort` at exit 0. +`pause_step` is null on the finalizing row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. The DNS skip is a finished command, not a pause — the wizard prints its summary and exits 0 — so it carries no code, and it is the row to remember when the `paused` class contains no DNS traffic. A Ctrl-C at a prompt _before_ the production instance exists is not any of these either — there is no state to preserve, so it stays a plain `abort` at exit 0 (a Ctrl-C while a request is in flight is the signal handler's, at exit 130). `stage` is the state the deploy was in when the run ended, on every `deploy` and `deploy status` event: the same value the status report's `state` field prints, so a wizard run and a `clerk deploy status` run a second later agree about the same deploy. It is the deploy's state, not the wizard's position. On a fresh deploy the DNS handoff comes before OAuth setup, so someone who skips a provider is at `domain_pending` with `pause_step: "oauth"`; `oauth_pending` there would contradict the status command. One value per run, the last one observed. diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 443f3cf9f..244dab0a0 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -3099,6 +3099,22 @@ describe("deploy", () => { // read fails, so the user can retry from the screen. That is not an // observation: recording `domain_pending` from it would file a network // blip as a DNS stall. + // The only stage a resume can establish without a domain read: Clerk + // listed no production domain, so the deploy is still provisioning one. + test("a resume with no production domain yet records domain_provisioning", async () => { + await linkedProject({ + instances: { development: "ins_dev_123", production: "ins_prod_123" }, + }); + mockIsAgent.mockReturnValue(false); + mockListApplicationDomains.mockResolvedValue({ data: [] }); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBe("domain_provisioning"); + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + test("a resume whose domain read failed records no stage when DNS is then skipped", async () => { await linkedProject({ instances: { development: "ins_dev_123", production: "ins_prod_123" }, diff --git a/packages/cli-core/src/commands/deploy/status.test.ts b/packages/cli-core/src/commands/deploy/status.test.ts index 8dfcd5abc..3d659ba37 100644 --- a/packages/cli-core/src/commands/deploy/status.test.ts +++ b/packages/cli-core/src/commands/deploy/status.test.ts @@ -338,8 +338,11 @@ describe("recording observations", () => { }); // Production configuration and domain status are read together, and each is -// recorded the moment it succeeds: a failure in one must not discard what the -// other observed, and the recording must not race the telemetry send. +// recorded the moment it succeeds, so a failure in one does not discard what +// the other observed. The reads fail fast: a read that finishes only after +// the event is built is dropped, never misrecorded. These mocks settle +// immediately, so the recording lands before the send — a tripwire for that +// ordering, not a guarantee the code makes. describe("resolveLiveDeploySnapshot records each read as it succeeds", () => { const serverError = () => new PlapiError(500, JSON.stringify({ errors: [{ code: "server_error" }] }), "https://x"); diff --git a/packages/cli-core/src/commands/deploy/status.ts b/packages/cli-core/src/commands/deploy/status.ts index bdf17e511..b80b69fbc 100644 --- a/packages/cli-core/src/commands/deploy/status.ts +++ b/packages/cli-core/src/commands/deploy/status.ts @@ -310,7 +310,12 @@ export async function resolveLiveDeploySnapshot( if (read.live) recordDomainObservation(deployComponentStatusFromDomainStatus(read.status)); return read; }); - const [productionConfig, { status: deployStatus, live }] = await settleBeforeRejecting([ + // Fail-fast on purpose: a `.then` on the slower read may still land + // after the run has built its event, in which case that observation is + // dropped — never wrong, just absent. Waiting for the slower read to + // settle would hold a real error behind a hanging request, and nothing + // bounds how long that is. + const [productionConfig, { status: deployStatus, live }] = await Promise.all([ configRead, statusRead, ]); @@ -347,24 +352,6 @@ export async function resolveLiveDeploySnapshot( }; } -/** - * `Promise.all`, except a rejection waits for the other promises to settle - * before it propagates. Same result and the same winning error — the first - * to fail — only the throw is delayed until a `.then` attached to a slower - * promise has run. Without this, whether that `.then` lands before or after - * the run finalizes its telemetry would be a race. - */ -async function settleBeforeRejecting( - promises: T, -): Promise<{ -readonly [P in keyof T]: Awaited }> { - try { - return await Promise.all(promises); - } catch (error) { - await Promise.allSettled(promises); - throw error; - } -} - function resolvePendingStep( pendingOAuthDescriptor: OAuthProviderDescriptor | undefined, domainComplete: boolean, diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 4f1e884e8..ee6fd0ef0 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -767,10 +767,10 @@ describe("finalizeAndSendTelemetry", () => { ); }); - // Declared in M3 so the shape is fixed once; the wizard fills them in + // The deploy fields are on every event; a command that never observes them // later milestones. Null means never observed, and the warehouse reads it // that way — it must not arrive as `false` or as an absent key. - test("the fields later milestones fill are present and null", async () => { + test("pause_step and components are present and null on a command that never sets them", async () => { const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); expect(payload.pause_step).toBeNull(); expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index abf7feaa1..1cbbe6054 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -71,7 +71,7 @@ export type TelemetryResult = { * Where a `clerk deploy` run stopped when the user has something left to do. * Narrower than `stage`: on a fresh deploy the DNS handoff runs before OAuth * setup, so someone who skips a provider is at `stage: "domain_pending"` and - * `pauseStep: "oauth"`. Set by the deploy wizard (GROW-1233 item 2). + * `pauseStep: "oauth"`. Set by the deploy wizard (GROW-1233). */ export type TelemetryPauseStep = "dns" | "oauth"; @@ -79,7 +79,7 @@ export type TelemetryPauseStep = "dns" | "oauth"; * Per-component readiness at the time the run ended. `null` means never * observed — no successful status read established it — and must never be * read as `false`: a failed status call is not a DNS failure. Filled by the - * deploy wizard and `clerk deploy status` (GROW-1233 item 4). + * deploy wizard and `clerk deploy status` (GROW-1233). */ export type TelemetryComponents = { dns: boolean | null; From eed94ec24950963d34dbecf03cb35beb2d12b669 Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Wed, 23 Sep 2026 17:40:36 -0700 Subject: [PATCH 15/16] Build the event from a copy taken at finalization --- packages/cli-core/src/lib/telemetry.test.ts | 29 +++++++++++++++++-- packages/cli-core/src/lib/telemetry.ts | 6 +++- .../src/test/integration/telemetry.test.ts | 2 +- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index ee6fd0ef0..b55a1c443 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -571,6 +571,28 @@ describe("finalizeAndSendTelemetry", () => { }); }); + // A deploy read still in flight when the command failed can resolve while + // the send is awaiting its config reads. The event is built from a copy + // taken when finalization began, so that late write changes nothing. + test("a component written after finalization begins does not reach the event", async () => { + await markTelemetryNoticeShown(); + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + const posted: string[] = []; + globalThis.fetch = (async (_url: unknown, init?: { body?: string }) => { + posted.push(init?.body ?? ""); + return new Response("{}"); + }) as unknown as typeof fetch; + startCommandTelemetry(fakeCommand()); + + const sending = finalizeAndSendTelemetry({ outcome: "error", exitCode: 1 }); + setTelemetryDomainComponents({ dns: true, ssl: true, mail: true }); + await sending; + + expect(posted).toHaveLength(1); + const payload = JSON.parse(posted[0]!).events[0].payload; + expect(payload.components).toEqual({ dns: null, ssl: null, mail: null, oauth: null }); + }); + // A command that reports failure through `process.exitCode` never reaches // `telemetryResultForError`, so without a declaration the only thing the // soft-exit branch can say is "nonzero, therefore error". @@ -604,7 +626,8 @@ describe("finalizeAndSendTelemetry", () => { expect(payload.error_code).toBeNull(); }); - // The shape M7 extends: `clerk api` holds a code its own catch swallowed. + // The shape the status-based split below extends: `clerk api` holds a code + // its own catch swallowed. test("a declaration can carry an error code", () => { startCommandTelemetry(fakeCommand()); declareSoftExitOutcome("error", "api_not_found"); @@ -768,8 +791,8 @@ describe("finalizeAndSendTelemetry", () => { }); // The deploy fields are on every event; a command that never observes them - // later milestones. Null means never observed, and the warehouse reads it - // that way — it must not arrive as `false` or as an absent key. + // sends null. Null means never observed, and the warehouse reads it that + // way — it must not arrive as `false` or as an absent key. test("pause_step and components are present and null on a command that never sets them", async () => { const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); expect(payload.pause_step).toBeNull(); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 1cbbe6054..c7ffc180e 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -521,7 +521,11 @@ export async function finalizeAndSendTelemetry( ): Promise { if (finalized || !context) return; - const current = context; + // A copy, not the live context: the send awaits config reads before it + // builds the event, and a deploy read still in flight when the command + // failed could land in that window. The event says what was known when + // the command ended, whatever finishes afterwards. + const current = { ...context, components: { ...context.components } }; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), deadlineMs); try { diff --git a/packages/cli-core/src/test/integration/telemetry.test.ts b/packages/cli-core/src/test/integration/telemetry.test.ts index b9d0c4b96..aa978de15 100644 --- a/packages/cli-core/src/test/integration/telemetry.test.ts +++ b/packages/cli-core/src/test/integration/telemetry.test.ts @@ -167,7 +167,7 @@ test("an unfinished `deploy status` is recorded as incomplete, not an error", as }); // `clerk api` catches the API error to print its body, so the code reaches -// the event only through the soft-exit declaration. M3's test above proves a +// the event only through the soft-exit declaration. The `incomplete` test above proves a // declared outcome survives the real program; this proves a declared *code* // does, and covers the status split end to end — the unit tests model the // final step, this runs it. From c8ede3b8154018721228d1577215438280c89f9d Mon Sep 17 00:00:00 2001 From: Shane Kercheval Date: Fri, 25 Sep 2026 09:19:43 -0700 Subject: [PATCH 16/16] Address Wyatt's review on outcomes, doctor and docs --- .changeset/grow-1233-cli-deploy-telemetry.md | 2 +- .../cli-core/src/commands/api/index.test.ts | 15 +- packages/cli-core/src/commands/api/index.ts | 21 +- .../cli-core/src/commands/api/interactive.ts | 18 +- .../cli-core/src/commands/deploy/README.md | 6 +- .../src/commands/deploy/index.test.ts | 28 ++ .../cli-core/src/commands/deploy/index.ts | 15 +- .../cli-core/src/commands/deploy/state.ts | 7 +- .../src/commands/doctor/index.test.ts | 3 + .../cli-core/src/commands/doctor/index.ts | 36 +-- packages/cli-core/src/lib/telemetry.test.ts | 14 + packages/cli-core/src/lib/telemetry.ts | 239 +++--------------- 12 files changed, 147 insertions(+), 257 deletions(-) diff --git a/.changeset/grow-1233-cli-deploy-telemetry.md b/.changeset/grow-1233-cli-deploy-telemetry.md index 7860ebefb..264f38b8f 100644 --- a/.changeset/grow-1233-cli-deploy-telemetry.md +++ b/.changeset/grow-1233-cli-deploy-telemetry.md @@ -4,6 +4,6 @@ Record `clerk deploy status` on an unfinished deploy as incomplete rather than an error in usage telemetry, and give the ways a `clerk deploy` run can end their own error codes — a skipped step, an interrupted prompt and a wait on Clerk's provisioning were previously indistinguishable. Every `clerk deploy` and `clerk deploy status` event now also records the state the deploy was in when the run ended, so a run that stopped short says where, and which of DNS, SSL, email DNS and OAuth had been verified at that point — recorded only from a read that actually succeeded, so a failed status call is never reported as a failed check. Output and exit codes are unchanged. -`clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. +`clerk doctor` now names the check that crashed instead of printing an anonymous "Check crashed" line (which `--json` labelled "Unknown check"), says the crash is a bug in the CLI instead of reporting issues with your integration, and reports a crashed check as `doctor_check_crashed` rather than `doctor_failed`, so a bug in the CLI is distinguishable from a real problem with your integration. Its `--json` results carry `crashed: true` on that check. The exit code is unchanged. `clerk api`, `clerk users create` and `clerk mcp install --json` failures now carry an error code in usage telemetry — each prints the failure itself instead of throwing, which used to leave the event with a bare error. An API response with a Clerk error code records that code; one without is recorded by its HTTP status as `api_rate_limited` (429), `api_not_found` (a 404 on a path the person typed), `cli_endpoint_not_found` (a 404 on a path the CLI built, from its endpoint catalog or a hardcoded route), `api_client_error` (other 4xx) or `api_error` (5xx). `mcp install --json` records the same code human mode already did. Nothing printed changes and exit codes are unchanged. diff --git a/packages/cli-core/src/commands/api/index.test.ts b/packages/cli-core/src/commands/api/index.test.ts index 95f408bdf..e79f86ea3 100644 --- a/packages/cli-core/src/commands/api/index.test.ts +++ b/packages/cli-core/src/commands/api/index.test.ts @@ -886,8 +886,15 @@ describe("api command", () => { describe("what telemetry records as the error code", () => { const clerkBody = (code: string) => JSON.stringify({ errors: [{ code, message: "" }] }); - function recordedFor(endpoint: string, options: Record = {}) { - return captureTelemetryPayload("api", () => runApi(endpoint, options)); + function recordedFor( + endpoint: string, + options: Record = {}, + caller: { userSuppliedPath?: boolean } = {}, + ) { + return captureTelemetryPayload("api", async () => { + const { api } = await import("./index.ts"); + await api(endpoint, undefined, options, caller); + }); } test("a Clerk error code in the response body", async () => { @@ -944,13 +951,13 @@ describe("api command", () => { // filed with the typed ones. test("an uncoded 404 on a catalog endpoint is cli_endpoint_not_found", async () => { stubFetch(async () => new Response("404 page not found", { status: 404 })); - const { payload } = await recordedFor("/organization_role", { userSuppliedPath: false }); + const { payload } = await recordedFor("/organization_role", {}, { userSuppliedPath: false }); expect(payload.outcome).toBe("error"); expect(payload.exit_code).toBe(1); expect(payload.error_code).toBe("cli_endpoint_not_found"); // A coded 404 names the resource, whoever wrote the path. stubFetch(async () => new Response(clerkBody("resource_not_found"), { status: 404 })); - const coded = await recordedFor("/users/bad_id", { userSuppliedPath: false }); + const coded = await recordedFor("/users/bad_id", {}, { userSuppliedPath: false }); expect(coded.payload.error_code).toBe("resource_not_found"); }); diff --git a/packages/cli-core/src/commands/api/index.ts b/packages/cli-core/src/commands/api/index.ts index 1bd1aada2..0b105efd7 100644 --- a/packages/cli-core/src/commands/api/index.ts +++ b/packages/cli-core/src/commands/api/index.ts @@ -27,15 +27,6 @@ export interface ApiOptions { fapi?: boolean; dryRun?: boolean; yes?: boolean; - /** - * Internal, not a flag. Who wrote the request path, for telemetry's 404 - * classification. Unset means the command line, so the person — safe only - * while Commander's registration and the interactive builder are the only - * callers. A caller that builds its own path must pass false, as the - * builder does for an endpoint chosen from the CLI's own catalog, so a 404 - * on it is recorded as the CLI's failure. - */ - userSuppliedPath?: boolean; } const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); @@ -73,10 +64,16 @@ async function resolveApiTarget( return { baseUrl, runRequest: async (req) => bapiRequest({ ...req, secretKey, baseUrl }) }; } +/** + * `userSuppliedPath` says who wrote the request path, for telemetry's 404 + * classification. The default is the command line, so the person; a caller + * that builds its own path passes false. + */ export async function api( endpoint: string | undefined, filter: string | undefined, options: ApiOptions, + { userSuppliedPath = true }: { userSuppliedPath?: boolean } = {}, ): Promise { const nested = isInsideGutter(); if (!nested) intro("Calling Clerk API"); @@ -165,7 +162,7 @@ export async function api( log.info(`If the endpoint path was a guess, search with: clerk api ls ${scope}`); } // Handled here, so telemetry never sees the throw it would classify. - declareSoftExitError(error, { userSuppliedPath: options.userSuppliedPath ?? true }); + declareSoftExitError(error, { userSuppliedPath }); process.exitCode = 1; closeStatus = "failed"; return; @@ -322,5 +319,7 @@ export function registerApi(program: Program): void { description: "GET the public FAPI environment payload", }, ]) - .action(api); + // Wrapped because Commander passes the Command itself as a fourth argument, + // which must not land in `api`'s caller options. + .action(async (endpoint, filter, options) => api(endpoint, filter, options)); } diff --git a/packages/cli-core/src/commands/api/interactive.ts b/packages/cli-core/src/commands/api/interactive.ts index 40e14c9c4..a7bdaaef7 100644 --- a/packages/cli-core/src/commands/api/interactive.ts +++ b/packages/cli-core/src/commands/api/interactive.ts @@ -100,14 +100,18 @@ export async function apiInteractive(options: ApiOptions): Promise { // 7. Delegate to the main api handler const { api } = await import("./index.ts"); - await api(resolvedPath, undefined, { - ...options, - method: endpoint.method, - data: body, - yes: true, // skip double-confirmation + await api( + resolvedPath, + undefined, + { + ...options, + method: endpoint.method, + data: body, + yes: true, // skip double-confirmation + }, // The endpoint is the catalog's, so a bare 404 is recorded as the CLI's. // A typed parameter goes into the route as typed, so a malformed value // (a slash, a dot segment) can be misfiled the same way; accepted as rare. - userSuppliedPath: false, - }); + { userSuppliedPath: false }, + ); } diff --git a/packages/cli-core/src/commands/deploy/README.md b/packages/cli-core/src/commands/deploy/README.md index 2eb0da6af..da251ca46 100644 --- a/packages/cli-core/src/commands/deploy/README.md +++ b/packages/cli-core/src/commands/deploy/README.md @@ -73,18 +73,22 @@ Four ways a run ends with the deploy unfinished and nothing broken. All four exi | Ending | `outcome` | `error_code` | `pause_step` | Exit | | ------------------------------------------------------------------- | --------- | ------------------- | ------------------------ | ---- | | The user skipped an OAuth provider | `error` | `deploy_paused` | the step they stopped on | 1 | -| The user interrupted a prompt after the production instance existed | `error` | `deploy_cancelled` | the step they stopped on | 130 | +| The user interrupted a prompt after the production instance existed | `abort` | `deploy_cancelled` | the step they stopped on | 130 | | Every DNS component verified, Clerk still provisioning | `error` | `deploy_finalizing` | null | 1 | | The user chose "skip" at DNS verification | `success` | null | null | 0 | `pause_step` is null on the finalizing row on purpose: nobody stopped there, the deploy is waiting on Clerk, and recording `dns` would count a drop-off that never happened. The DNS skip is a finished command, not a pause — the wizard prints its summary and exits 0 — so it carries no code, and it is the row to remember when the `paused` class contains no DNS traffic. A Ctrl-C at a prompt _before_ the production instance exists is not any of these either — there is no state to preserve, so it stays a plain `abort` at exit 0 (a Ctrl-C while a request is in flight is the signal handler's, at exit 130). +The interrupted row is `abort`, like a Ctrl-C anywhere else, so the same keypress is not counted as an error when it happens to land on a prompt; `deploy_cancelled` and `pause_step` still say which prompt. + `stage` is the state the deploy was in when the run ended, on every `deploy` and `deploy status` event: the same value the status report's `state` field prints, so a wizard run and a `clerk deploy status` run a second later agree about the same deploy. It is the deploy's state, not the wizard's position. On a fresh deploy the DNS handoff comes before OAuth setup, so someone who skips a provider is at `domain_pending` with `pause_step: "oauth"`; `oauth_pending` there would contradict the status command. One value per run, the last one observed. It is null when no reliable state was established by the time the run ended, and that null is a different answer from `not_started`. That covers a run that failed before reading anything — not linked, a failed sign-in, an API error on the first read — and two cases where a state was invalidated or never observed: a resume whose domain read failed and substituted an all-pending status so the user could retry from the screen, where the user then skipped verification; and a fresh run whose create call answered that an instance already exists, after which the resume could not read it. Two states are known without a status read: a fresh deploy starts at `not_started`, and a newly created instance is at `domain_pending` the moment Clerk returns it with a domain (`domain_provisioning` if it did not). Every other value comes from a read that succeeded. `components` says which of the four pieces were verified when the run ended: `dns`, `ssl` and `mail` from the domain-status response, `oauth` from whether every required provider has production credentials — the same facts the status report's `domainStatus` and `oauth.complete` print. Each is `true`, `false` or null, and null means never observed, which is a different answer from `false`: a failed status call is not a DNS failure. The four come from two reads, so they are two observations. A domain poll rewrites the first three and leaves `oauth` as it was; a production-configuration read or a credential save rewrites `oauth` and leaves the other three. Within a group the last observation wins. A read that never happened, or a substituted one, writes nothing, and each read is recorded the moment it succeeds, so a failure in the other read does not discard it. So a resume whose domain read failed sends `oauth` from its configuration read and null for the other three; a `clerk deploy status` whose domain read failed does the same, with a null stage, since no state was established. On a fresh deploy, `oauth` is written only once every required credential is saved (`true`), because until then nothing has read the production configuration — a save proves only the provider it saved. A fresh run that pauses part-way through OAuth setup therefore sends null for `oauth` even though `clerk deploy status` on the same deploy would read the configuration and say `false`; that is the one place the two are allowed to differ, and null rather than an assumed `false` is deliberate. A fresh run that saves every provider's credentials and skips DNS verification sends `oauth: true` with the other three null. `oauth` reflects the CLI's required-provider rule as it stands; when GROW-1236 changes that rule, this value follows, because it is computed from the same report. +What the warehouse depends on, as of data-platform#604, so a change here is checked there too. Its classifier reads `outcome` before anything else, which is why a command may not declare `success` for a nonzero exit. Its payload contract test accepts exactly the five deploy `stage` values on `deploy run` and `deploy status`, which is why a finished deploy is `complete` and never the shared `done`. It alarms on a missing `pause_step` only for `deploy_paused` and `deploy_cancelled`, so a new ending that should carry a step needs adding there. `pause_step` and `components` are top-level payload keys, null on other commands, because the staging model reads those exact paths. + Agent mode is detected via the mode system (`src/mode.ts`), which checks in priority order: 1. `--mode` CLI flag diff --git a/packages/cli-core/src/commands/deploy/index.test.ts b/packages/cli-core/src/commands/deploy/index.test.ts index 244dab0a0..c094287c1 100644 --- a/packages/cli-core/src/commands/deploy/index.test.ts +++ b/packages/cli-core/src/commands/deploy/index.test.ts @@ -2653,6 +2653,7 @@ describe("deploy", () => { const { payload } = await deployTelemetry(async () => runDeploy({})); + expect(payload.outcome).toBe("abort"); expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("oauth"); expect(payload.stage).toBe("oauth_pending"); @@ -2720,6 +2721,7 @@ describe("deploy", () => { const { payload } = await deployTelemetry(async () => runDeploy({})); + expect(payload.outcome).toBe("abort"); expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("dns"); expect(payload.stage).toBe("domain_pending"); @@ -2737,6 +2739,7 @@ describe("deploy", () => { const { payload } = await deployTelemetry(async () => runDeploy({})); + expect(payload.outcome).toBe("abort"); expect(payload.error_code).toBe(ERROR_CODE.DEPLOY_CANCELLED); expect(payload.pause_step).toBe("dns"); expect(payload.stage).toBe("domain_pending"); @@ -2995,6 +2998,31 @@ describe("deploy", () => { expect(payload.stage).toBeNull(); }); + // Create says an instance exists, then the refresh finds no production + // instance id: the run never identified an instance, so it claims no + // state for one. + test("a create conflict whose refresh finds no instance records no stage", async () => { + await linkedProject(); + mockIsAgent.mockReturnValue(false); + mockCreateConflict(); + mockFetchApplication.mockResolvedValue({ + application_id: "app_xyz789", + name: "my-saas-app", + instances: [ + { + instance_id: "ins_dev_123", + environment_type: "development", + publishable_key: "pk_test_123", + }, + ], + }); + + const { payload } = await deployTelemetry(async () => runDeploy({})); + + expect(payload.outcome).toBe("success"); + expect(payload.stage).toBeNull(); + }); + test("a create conflict whose resume reads the deploy records what it observed", async () => { await linkedProject(); mockIsAgent.mockReturnValue(false); diff --git a/packages/cli-core/src/commands/deploy/index.ts b/packages/cli-core/src/commands/deploy/index.ts index c3ea03684..224742b68 100644 --- a/packages/cli-core/src/commands/deploy/index.ts +++ b/packages/cli-core/src/commands/deploy/index.ts @@ -75,7 +75,7 @@ import { type DiscoveredOAuthProviders, type LiveDeploySnapshot, } from "./status.ts"; -import { clearTelemetryStage } from "../../lib/telemetry.ts"; +import { setTelemetryStage } from "../../lib/telemetry.ts"; import { resolveActiveReportState, type OAuthSetupFacts } from "./report-state.ts"; import { recordDeployObservation, @@ -186,11 +186,9 @@ async function startNewDeploy(ctx: DeployContext): Promise { const productionOrExists = await createProductionInstance(ctx, domain); if (productionOrExists === "exists") { - // An observation that disproves the stage without establishing a new one: - // `not_started` is now false, and whether that instance has a domain, or - // how far it got, is unknown until the resume below reads it. If that read - // fails, or substitutes, the run ends with no stage rather than a false one. - clearTelemetryStage(); + // `not_started` is now false and the resume has not read anything yet, so + // the run sends no stage unless the resume below observes one. + setTelemetryStage(null); log.blank(); log.info( "A production instance already exists for this application. Resuming the existing deploy.", @@ -286,7 +284,10 @@ async function reconcileExistingDeploy(ctx: DeployContext): Promise { const snapshot = await resolveLiveDeploySnapshot(ctx); if (!snapshot) { - recordDeployStage("domain_provisioning"); + // No snapshot also means no production instance id, which only happens when + // a create said one exists and the refresh could not find it. The run knows + // nothing then, so it records nothing. + if (ctx.productionInstanceId) recordDeployStage("domain_provisioning"); log.blank(); log.info("A production instance exists, but Clerk did not return a production domain yet."); log.info("Run `clerk deploy` again after the domain is available from the API."); diff --git a/packages/cli-core/src/commands/deploy/state.ts b/packages/cli-core/src/commands/deploy/state.ts index 6ec30b3a4..cbc395915 100644 --- a/packages/cli-core/src/commands/deploy/state.ts +++ b/packages/cli-core/src/commands/deploy/state.ts @@ -58,12 +58,7 @@ const PAUSE_REASONS: Record< { code: ErrorCode; exitCode: typeof EXIT_CODE.GENERAL | typeof EXIT_CODE.SIGINT; - /** - * Whether the person stopped at a step. The CLI's own call: the warehouse - * validates the step value but only alarms on its absence for the codes - * it knows (as of data-platform#604), so a new reason that should carry a - * step also needs adding there. - */ + /** Whether the person stopped at a step, so `pause_step` is recorded. */ recordsPauseStep: boolean; } > = { diff --git a/packages/cli-core/src/commands/doctor/index.test.ts b/packages/cli-core/src/commands/doctor/index.test.ts index ee7da28d5..a2d6d7de8 100644 --- a/packages/cli-core/src/commands/doctor/index.test.ts +++ b/packages/cli-core/src/commands/doctor/index.test.ts @@ -101,6 +101,9 @@ describe("doctor", () => { const error = await runDoctor(); expect(error?.code).toBe(ERROR_CODE.DOCTOR_CHECK_CRASHED); + // The message must not send the reader to their project for a CLI bug. + expect(error?.message).toContain("bug in the Clerk CLI"); + expect(error?.message).not.toContain("issues with your Clerk integration"); }); // A crash outranks a finding: the run can no longer claim to have checked diff --git a/packages/cli-core/src/commands/doctor/index.ts b/packages/cli-core/src/commands/doctor/index.ts index f1574c68d..5b2d06856 100644 --- a/packages/cli-core/src/commands/doctor/index.ts +++ b/packages/cli-core/src/commands/doctor/index.ts @@ -84,24 +84,18 @@ async function runChecks(ctx: DoctorContext): Promise { } /** - * What to throw for a set of results that includes a failure. A crashed check - * and a real finding are both exit 1, but they send the reader to different - * places — one is a CLI bug, the other is the user's integration — and a - * single code left them indistinguishable in telemetry and on screen. - * - * Decided from one result set. After `--fix`, that is the verify pass alone: - * it re-runs every check, so it is the complete answer and the screen the - * user last saw. The cost is that a first-pass crash the verify pass does not - * reproduce is recorded nowhere — a transient one is superseded by whatever - * durable finding remained, which is why `doctor_check_crashed` rows can be - * rarer than crashes people report. + * The error for a set of results that includes a failure. A crashed check is a + * CLI bug, not a problem with the user's project, so both the message and the + * code say so. After `--fix` this is decided from the verify pass alone. */ -function failureCodeFor( - results: CheckResult[], -): typeof ERROR_CODE.DOCTOR_CHECK_CRASHED | typeof ERROR_CODE.DOCTOR_FAILED { - return results.some((r) => r.crashed) - ? ERROR_CODE.DOCTOR_CHECK_CRASHED - : ERROR_CODE.DOCTOR_FAILED; +function failureFor(results: CheckResult[], findingsMessage: string): CliError { + const crashed = results.some((r) => r.crashed); + return new CliError( + crashed + ? "A doctor check crashed. This is a bug in the Clerk CLI, not your project; see the check marked as crashed above." + : findingsMessage, + { code: crashed ? ERROR_CODE.DOCTOR_CHECK_CRASHED : ERROR_CODE.DOCTOR_FAILED }, + ); } function printResults(results: CheckResult[], options: DoctorOptions): void { @@ -176,9 +170,7 @@ export async function doctor(options: DoctorOptions = {}): Promise { const hasVerifyFailure = verifyResults.some((r) => r.status === "fail"); if (hasVerifyFailure) { - throw new CliError("Some checks still failing after auto-fix", { - code: failureCodeFor(verifyResults), - }); + throw failureFor(verifyResults, "Some checks still failing after auto-fix"); } await outro("All checks passing"); return; @@ -187,9 +179,7 @@ export async function doctor(options: DoctorOptions = {}): Promise { const hasFailure = allResults.some((r) => r.status === "fail"); if (hasFailure) { - throw new CliError("Doctor found issues with your Clerk integration", { - code: failureCodeFor(allResults), - }); + throw failureFor(allResults, "Doctor found issues with your Clerk integration"); } await outro("All checks passing"); } diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index b55a1c443..66def2591 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -142,6 +142,20 @@ describe("telemetryResultForError", () => { expect(telemetryResultForError(new CliError("nope")).errorCode).toBe("cli_error"); }); + // A cancelled deploy prompt is Ctrl-C, the same keypress the interrupt path + // records as an abort; the code still says which prompt. + test("maps a CliError that exits 130 to abort, keeping its code", () => { + const error = new CliError("paused", { + code: ERROR_CODE.DEPLOY_CANCELLED, + exitCode: EXIT_CODE.SIGINT, + }); + expect(telemetryResultForError(error)).toEqual({ + outcome: "abort", + exitCode: EXIT_CODE.SIGINT, + errorCode: "deploy_cancelled", + }); + }); + test("maps ApiError (code is null for a non-JSON body → api_error fallback)", () => { const error = new ApiError(500, "boom"); expect(telemetryResultForError(error)).toEqual({ diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index c7ffc180e..28155f438 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -37,27 +37,17 @@ import { getMode } from "../mode.ts"; import { CURRENT_VERSION, IS_DEV_BUILD } from "./version.ts"; /** - * What happened to the command, not to the thing it acted on. - * - * `incomplete` says the command ran and the thing it reports on is not - * finished — nobody is being asked to do anything, and nothing failed. Only - * `clerk deploy status` sends it, and only by declaring it (see - * {@link declareSoftExitOutcome}); it is never a mapping of nonzero exits. - * - * `success` is not "the deploy is done" either: `clerk deploy` under an agent - * prints a status report and exits 0 with nothing started. How far a deploy - * got is `stage` and `components`, never `outcome`. + * What happened to the command, not to the thing it acted on. `incomplete` + * means the command answered and the thing it reports on is not finished; only + * `clerk deploy status` sends it, by declaring it. How far a deploy got is + * `stage` and `components`, never `outcome`. */ export type TelemetryOutcome = "success" | "error" | "abort" | "incomplete"; /** - * What a command may declare for itself on the soft-exit path. - * - * Deliberately narrower than {@link TelemetryOutcome}. `success` is excluded - * because the warehouse classifies a row by `outcome` before it looks at - * anything else (as of data-platform#604), so declaring it on a run that then - * exits nonzero would file a failure as a success. `abort` is excluded because - * it belongs to the interrupt path, which reports itself. + * What a command may declare for a nonzero soft exit. Not `success`, which + * would file a failure as a success, and not `abort`, which the interrupt path + * reports itself. */ export type SoftExitOutcome = "incomplete" | "error"; @@ -67,19 +57,12 @@ export type TelemetryResult = { errorCode?: string; }; -/** - * Where a `clerk deploy` run stopped when the user has something left to do. - * Narrower than `stage`: on a fresh deploy the DNS handoff runs before OAuth - * setup, so someone who skips a provider is at `stage: "domain_pending"` and - * `pauseStep: "oauth"`. Set by the deploy wizard (GROW-1233). - */ +/** The step a `clerk deploy` run stopped on when the person has something left to do. */ export type TelemetryPauseStep = "dns" | "oauth"; /** - * Per-component readiness at the time the run ended. `null` means never - * observed — no successful status read established it — and must never be - * read as `false`: a failed status call is not a DNS failure. Filled by the - * deploy wizard and `clerk deploy status` (GROW-1233). + * Per-component readiness when the run ended. `null` means never observed, + * which is not `false`: a failed status read is not a DNS failure. */ export type TelemetryComponents = { dns: boolean | null; @@ -116,21 +99,9 @@ export type TelemetryStage = | "token_exchange" | "store" | "first_application" - // `clerk deploy` and `clerk deploy status` - // - // Unlike the groups above, these are not control-flow positions: each is a - // state of the deploy itself, as `resolveActiveReportState` in - // `commands/deploy/report-state.ts` would compute it at that moment. So the stage - // a wizard run reports and the stage `clerk deploy status` reports a second - // later agree about the same deploy. One value per run — the last state - // observed, not every state the run passed through — and a run that ends - // before any state resolves sends null rather than defaulting: "never - // established" is a distinct answer from "not started". - // - // A finished deploy is `complete`, never the shared `done` marker below: - // the warehouse's payload contract test accepts exactly these five values - // on `deploy run` and `deploy status`, so `done` there trips it on every - // finished deploy. + // `clerk deploy` and `clerk deploy status`: the state of the deploy itself, + // the value `clerk deploy status` reports, not a control-flow position. A + // finished deploy is `complete`, never `done`; see commands/deploy/README.md. | "not_started" | "domain_provisioning" | "domain_pending" @@ -161,12 +132,7 @@ type TelemetryContext = { components: TelemetryComponents; }; -/** - * What a command wants recorded when it reports failure through - * `process.exitCode` rather than by throwing. The error code is optional - * because `clerk deploy status` has none to give: nothing was thrown, so - * there is no code, and `incomplete` is the whole answer. - */ +/** What a command wants recorded for its soft exit. `deploy status` declares no code: nothing was thrown. */ type SoftExitDeclaration = { outcome: SoftExitOutcome; errorCode?: string; @@ -289,48 +255,21 @@ export function startCommandTelemetry(actionCommand: TelemetryCommand): void { * an error-only dimension: a user declining the scaffold preview and a * failure inside the generator are both legible, and distinguishable. */ -export function setTelemetryStage(stage: TelemetryStage): void { +export function setTelemetryStage(stage: TelemetryStage | null): void { if (context) context.stage = stage; } -/** - * Forget the stage. For the one case where an observation disproves the - * stage last set without establishing a new one — a fresh deploy's create - * call answering that an instance already exists, in `commands/deploy/index.ts`, - * is the only caller. Not a general reset: a command that wants a different - * stage sets it. - */ -export function clearTelemetryStage(): void { - if (context) context.stage = null; -} - /** Read the stage a caller had set, so a nested flow can hand it back. */ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; } -/** - * Record the step a `clerk deploy` run stopped on. Set where the pause itself - * is constructed, which is the one place that knows both that the run is - * stopping and which step it stopped on — a caller that set it earlier would - * have to unset it on every path that then carried on. - * - * Only set it for a step the *person* stopped on. A wait on Clerk's backend - * ends the run at no step at all, and leaving the last step in place there - * would count it as a drop-off nobody made. - */ +/** Record the step a `clerk deploy` run stopped on. Only for a step the person stopped on. */ export function setTelemetryPauseStep(step: TelemetryPauseStep): void { if (context) context.pauseStep = step; } -/** - * Record what a successful domain-status read said about DNS, SSL and email - * DNS. Only ever called with a live read's answer: the wizard's substituted - * "everything pending" status and its fresh-run placeholder are not - * observations, and recording either would file a network blip as a DNS - * failure. Leaves `oauth` alone — it comes from a different read, and a - * domain poll must not erase a good OAuth observation or re-send a stale one. - */ +/** Record DNS, SSL and email DNS from a successful domain-status read. Leaves `oauth` alone. */ export function setTelemetryDomainComponents(status: { dns: boolean; ssl: boolean; @@ -345,59 +284,21 @@ export function setTelemetryDomainComponents(status: { }; } -/** - * Record whether every required OAuth provider has production credentials, - * from a successful production-configuration read or a credential save. - * "Required" is the CLI's rule as it stands — the providers enabled in - * development that the wizard knows how to configure. GROW-1236 changes that - * rule to read production configuration; this value follows automatically, - * because it is computed from the same report. Leaves the domain group alone. - */ +/** Record whether every required OAuth provider has production credentials. Leaves the domain components alone. */ export function setTelemetryOAuthComplete(complete: boolean): void { if (context) context.components = { ...context.components, oauth: complete }; } /** - * Declare what this run should be recorded as when it ends by setting - * `process.exitCode` instead of throwing. - * - * Commands that catch their own failure never reach `telemetryResultForError`, - * so without this the soft-exit branch in `cli-program.ts` can only say - * "nonzero, therefore error". That is wrong in both directions: `clerk deploy - * status` exits 1 on a deploy that simply is not finished, and `clerk api` - * exits 1 holding an error code a throw would have recorded (see - * {@link declareSoftExitError} for that side). - * - * Why this is a declaration and not a rule about exit codes: the exit code is - * a per-command transport detail — 1 means "not done" from `deploy status` - * and "request failed" from `api` — so only the command knows what its own - * nonzero exit meant. A general mapping would relabel every command at once. - * - * Ignored when the run throws: a thrown error is the more specific fact, and - * `runProgram` classifies it through {@link telemetryResultForError}. - * - * Two rules for callers: - * - * - **The last call wins.** Call this once, with the fact you want recorded. - * A command that aggregates failures across several targets and means to - * report the first one must select that error before calling, not call from - * inside its loop — which would record the last target's failure instead, - * with no test failing and telemetry naming the wrong thing. - * - **It applies to whatever nonzero code the run ends with,** not only the - * one in force when it was called. Declare it under the same condition that - * sets the exit code, so the two cannot diverge. + * Declare how a run that exits nonzero without throwing should be recorded. + * Last call wins; ignored if the run throws or exits 0. Declare it under the + * same condition that sets the exit code. */ export function declareSoftExitOutcome(outcome: SoftExitOutcome, errorCode?: string): void { if (context) context.softExit = { outcome, errorCode }; } -/** - * How a run that set `process.exitCode` and returned is recorded. Honors a - * declaration only on a nonzero exit: a command that declared an outcome and - * then succeeded anyway (a retry that worked, a later branch clearing the - * code) is a success, and reporting the stale declaration would invent a - * failure the user never saw. - */ +/** How a run that set `process.exitCode` and returned is recorded. A declaration applies only to a nonzero exit. */ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { if (exitCode === EXIT_CODE.SUCCESS) return { outcome: "success", exitCode }; const declared = context?.softExit; @@ -410,38 +311,12 @@ export function telemetryResultForSoftExit(exitCode: number): TelemetryResult { } /** - * Declare a failure the command caught and reported itself, carrying the code - * a throw would have. - * - * `clerk api`, `clerk users create` and `clerk mcp install --json` each catch - * their own error for a reason that stays as it is — the raw response body has - * to reach stdout for piping, or a second JSON document must not follow the - * first — and set the exit code instead. `telemetryResultForError` then never - * runs, and the code the error was holding is lost. This is the same - * classification, applied where the error is still in hand: a `CliError` - * keeps its named code, anything unrecognised is `unexpected_error`, so - * `mcp install --json` records what human mode records when it rethrows. - * - * The one difference from a throw: an `ApiError` with no parsed Clerk code is - * split by HTTP status rather than collapsed onto `api_error`. See - * {@link uncodedApiErrorCode} for why. Thrown `ApiError`s keep `api_error` - * because that code is on the warehouse's reviewed failure list as it is. - * - * `userSuppliedPath` says who wrote the request path, which only the call - * site knows and which decides what an uncoded 404 means: a person's typo, or - * the CLI asking for a route the API does not serve. It is required rather - * than defaulted so a new call site cannot mis-file a 404 by omission: the - * BAPI commands build their own paths and pass false; `clerk api` passes true - * for a path typed on the command line and false for one its interactive - * builder chose from the endpoint catalog. - * - * Call it under the same condition that sets the exit code, and with the - * error the run means to report — the last-call-wins rule on - * {@link declareSoftExitOutcome} applies. Never hand it a `UserAbortError`: - * a declaration cannot express an abort, so it would be recorded as - * `unexpected_error`. A command that prompts inside a caught section must let - * the abort throw instead. (No caller can reach this today; the MCP client - * picker runs before any client is settled.) + * Declare a failure the command caught and reported itself, with the code a + * throw would have carried. An uncoded `ApiError` is split by status (see + * {@link uncodedApiErrorCode}); thrown ones keep `api_error`. `userSuppliedPath` + * says who wrote the request path, which decides whether an uncoded 404 is the + * person's or the CLI's. Never pass a `UserAbortError`: it would be recorded as + * `unexpected_error`. */ export function declareSoftExitError(error: unknown, options: { userSuppliedPath: boolean }): void { const code = @@ -452,34 +327,13 @@ export function declareSoftExitError(error: unknown, options: { userSuppliedPath } /** - * An API response with no Clerk error code in its body has one HTTP status and - * no single meaning, so each code names exactly what was observed and nothing - * more. Telemetry carries no status and no endpoint, so this split is the only - * thing that makes the uncoded population measurable. - * - * - 429 → `api_rate_limited`, not the existing `too_many_requests`: that one - * arrives parsed from Clerk's error body, so it means Clerk itself said so. - * An uncoded 429 means no body said so — an empty body or an unexpected - * shape from Clerk parses the same as a proxy's answer, so the origin is - * unknown. Merging the two would erase the only distinction observable at - * the point of record. - * - 404 with a path the person typed → `api_not_found`: the path did not - * reach a Clerk route, and the person chose it. The hint `clerk api` - * prints on this branch is a heuristic, so the code claims the status and - * who wrote the path, not the cause. - * - 404 with a path the CLI built → `cli_endpoint_not_found`: the CLI asked - * for a route and nothing served it — a stale endpoint catalog, a hardcoded - * path the API dropped, or something in front of the API answering for it - * (`CLERK_BACKEND_API_URL` is overridable), the same ambiguity the 5xx - * bullet carries. The warehouse counts it as a failure. Kept apart from - * `api_not_found` because the same status means opposite things depending - * on who wrote the path, and the row cannot say which afterwards. - * - other 4xx → `api_client_error`: 400, 401 and 403 collapsed. Cause and - * frequency unknown; the status cannot be recovered afterwards, so no - * finer mapping is promised. - * - anything else → `api_error`: a 5xx is a failed request whoever caused it, - * Clerk or a customer's proxy — the same ambiguity every thrown `ApiError` - * carries today. + * The code for an API response with no Clerk error code, from its status alone: + * - 429: `api_rate_limited`, kept apart from `too_many_requests`, which Clerk's + * own error body names; an uncoded 429's origin is unknown. + * - 404: `api_not_found` on a path the person typed, `cli_endpoint_not_found` + * on one the CLI built (a stale catalog, a dropped route, or a proxy). + * - other 4xx: `api_client_error`. + * - anything else: `api_error`. */ function uncodedApiErrorCode(status: number, userSuppliedPath: boolean): string { if (status === 429) return "api_rate_limited"; @@ -493,7 +347,10 @@ export function telemetryResultForError(error: unknown): TelemetryResult { return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; } if (error instanceof CliError) { - return { outcome: "error", exitCode: error.exitCode, errorCode: error.code ?? "cli_error" }; + // Exit 130 is Ctrl-C (a cancelled deploy prompt), the same keypress the + // interrupt path records as an abort; the code still says which. + const outcome = error.exitCode === EXIT_CODE.SIGINT ? "abort" : "error"; + return { outcome, exitCode: error.exitCode, errorCode: error.code ?? "cli_error" }; } if (error instanceof ApiError) { return { outcome: "error", exitCode: EXIT_CODE.GENERAL, errorCode: error.code ?? "api_error" }; @@ -521,10 +378,8 @@ export async function finalizeAndSendTelemetry( ): Promise { if (finalized || !context) return; - // A copy, not the live context: the send awaits config reads before it - // builds the event, and a deploy read still in flight when the command - // failed could land in that window. The event says what was known when - // the command ended, whatever finishes afterwards. + // A copy, so a read that finishes after the command ended cannot change + // the event while the send is still building it. const current = { ...context, components: { ...context.components } }; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), deadlineMs); @@ -580,19 +435,9 @@ async function buildAndSend( outcome: result.outcome, exit_code: result.exitCode, error_code: result.errorCode ?? null, - // `stage` is shared (init, login and deploy each write their own group). - // `pause_step` and `components` are deploy's and ride on every other - // command's event as null members. They sit at the top level because the - // warehouse staging model already reads these exact paths (as of - // data-platform#604), so nesting them under a per-command key now would - // cost a warehouse change for no visible gain. That is a cost call, not - // a shape to copy: a command that needs its own structured detail can - // still add a namespaced object, with a contract-test arm to match. + // `pause_step` and `components` are deploy's; null on other commands. stage: current.stage, pause_step: current.pauseStep, - // Nested rather than four flat keys: it is one JSON path per component - // in the warehouse, and the group is obviously one thing. A null member - // means never observed — see TelemetryComponents. components: current.components, duration_ms: Date.now() - current.startedAt, machine_uuid: machineUuid,