feat(cli): report crashes in the CLI's own code to Checkly [ask]

.github/workflows/release.yml

@@ -89,6 +89,15 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      # Release builds report crashes in the CLI's own code to Checkly (see
+      # src/services/error-reporting.ts). The DSN is public by nature: it can only
+      # send events to one project. Builds without the file report nothing.
+      - name: Write the error reporting DSN for 'cli' package
+        if: ${{ vars.CHECKLY_CLI_ERROR_DSN != '' }}
+        env:
+          DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }}
+        run: jq -n --arg dsn "$DSN" '{dsn: $dsn}' > error-reporting.json
+        working-directory: packages/cli
       - name: Set version and publish prerelease for 'cli' package
         run: |
           pnpm version ${{ github.event.release.tag_name }}-prerelease-${{ env.SHORT_SHA }} --no-git-tag-version
@@ -158,6 +167,15 @@ jobs:
       # Ensure that the README is published with the package
       - run: rm -f packages/cli/README.md && cp README.md packages/cli
       - run: pnpm install --frozen-lockfile
+      # Release builds report crashes in the CLI's own code to Checkly (see
+      # src/services/error-reporting.ts). The DSN is public by nature: it can only
+      # send events to one project. Builds without the file report nothing.
+      - name: Write the error reporting DSN for 'cli' package
+        if: ${{ vars.CHECKLY_CLI_ERROR_DSN != '' }}
+        env:
+          DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }}
+        run: jq -n --arg dsn "$DSN" '{dsn: $dsn}' > error-reporting.json
+        working-directory: packages/cli
       - name: Set version and publish 'cli' package
         run: |
           pnpm version ${{ github.event.release.tag_name }} --no-git-tag-version

.gitignore

@@ -20,3 +20,6 @@ local
 **/checkly-github-report.md
 **/checkly-summary.md
 **/e2e/__tests__/fixtures/empty-project/e2e-test-project-*
+
+# Written by the release workflow; see packages/cli/src/services/error-reporting.ts
+packages/cli/error-reporting.json

CLAUDE.md

@@ -121,6 +121,7 @@ Source in `src/ai-context/`, built during `prepare`. Generates examples from fix
 - `CHECKLY_CLI_MODE` — force the CLI mode (`interactive`, `agent` or `ci`) instead of detecting it; an explicit `interactive` also lets `checkly login` show its code without a detected terminal (e2e harness, terminals Node doesn't recognise). The mode also sets the `x-checkly-source` header: `agent` reports `AGENT`, which changes how the backend classifies the requests (see `requestInterceptor` in `src/rest/api.ts`)
 - `CHECKLY_NO_BROWSER` — set to `1` so `checkly login` prints the login URL without trying to open a browser
 - `CHECKLY_NO_DELEGATION` — set to `1` to stop the CLI from handing every command to the project's own `node_modules/checkly` (searched upwards from the `--config`/`-c` file's directory, or the working directory) when that is a different install (`bin/launch.cjs`, `src/services/project-cli.ts`). Useful when running a dev checkout in a project that has its own `checkly`; constructs then fail to register unless the project resolves `checkly` to the same install
+- `CHECKLY_NO_ERROR_REPORTING` — set to `1` (or set `DO_NOT_TRACK=1`) to stop the CLI from reporting crashes in its own code to Checkly. Only release builds report: the release workflow writes the DSN from the `CHECKLY_CLI_ERROR_DSN` repository variable to `packages/cli/error-reporting.json`. Only TypeError, ReferenceError and RangeError raised in the CLI's own files are sent (also when wrapped as a `cause`, or printed by a command that catches it: `BaseCommand.finally` sends what `noteCliError` kept), once per run, without paths, credentials, arguments or environment (`src/services/error-reporting/`)
 - `CHECKLY_CACHE_DIR` — override the CLI's cache directory (embedded-package tarball downloads)
 - `CHECKLY_SKIP_NODE_VERSION_CHECK` — set to `1` to bypass the bin's hard Node version preflight (unsupported Node may then fail in unexpected ways); must be set in the shell environment — the preflight runs before `.env` is loaded
 - `CHECKLY_LOCKFILE_PRUNE` — set to `0` to disable pruning the bundled lockfile to the code bundle's contents; when a lockfile is bundled this also disables `bundle.packages.prune` (the manifest rewrite rolls back)

README.md

@@ -113,6 +113,33 @@ This provides your AI with documentation, best practices, and examples so it can
 
 Check out our [AI Agents & Coding Assistants docs](https://www.checklyhq.com/docs/integrations/ai/overview) for more best practices and examples.
 
+# Error reporting
+
+When the CLI crashes because of a bug in its own code, it sends one report about the crash to Checkly, so we can find
+and fix the bug. Only crashes inside the CLI's own files are reported (a `TypeError`, `ReferenceError` or
+`RangeError`). Errors in your own code, failed checks, API errors and network problems are never reported. Invalid
+configuration is normally shown as a clear message and not reported; if it makes the CLI crash instead, that crash is
+reported, because the CLI should have caught the mistake.
+
+A report contains:
+
+- the error type, its message and the stack trace, with paths reduced to the CLI's own files (your home directory,
+  project directory and file names are removed)
+- the CLI version, the command name (for example `deploy`), and whether the CLI runs in a terminal, in CI or for an
+  AI agent
+- your operating system, CPU architecture and Node.js version
+
+A report never contains your API key, account ID, command arguments, environment variables, computer name or the
+contents of your files. Credentials found in an error message are removed before sending.
+
+To turn error reporting off, set either environment variable:
+
+```bash
+export CHECKLY_NO_ERROR_REPORTING=1
+# or the cross-tool convention
+export DO_NOT_TRACK=1
+```
+
 
 ## Need help?
 

packages/cli/package.json

@@ -43,7 +43,8 @@
     "/assets",
     "/constructs.js",
     "/constructs.d.ts",
-    "/oclif.manifest.json"
+    "/oclif.manifest.json",
+    "/error-reporting.json"
   ],
   "oclif": {
     "bin": "checkly",
@@ -116,6 +117,7 @@
     "@oclif/core": "^5.1.9",
     "@oclif/plugin-help": "^7.0.5",
     "@oclif/plugin-warn-if-update-available": "^4.0.4",
+    "@sentry/core": "^11.10.0",
     "acorn": "^8.19.0",
     "acorn-walk": "^8.3.5",
     "archiver": "^8.0.0",

packages/cli/src/commands/baseCommand.ts

@@ -12,6 +12,8 @@ import { findStaleSkills } from '../services/skills.js'
 import { PackageJsonFile } from '../services/check-parser/package-files/package-json-file.js'
 import { detectNearestPackageJson } from '../services/check-parser/package-files/package-manager.js'
 import { isDevelopmentBuild } from '../services/project-cli.js'
+import { noteCliError, reportNotedCliError } from '../services/error-reporting/report.js'
+import { detectCliMode } from '../helpers/cli-mode.js'
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
 
@@ -195,7 +197,14 @@ export abstract class BaseCommand extends Command {
     if (err instanceof NotFoundError) {
       err.message = this.withNotFoundHint(err.message)
     }
-    // TODO: we can add Sentry here and log critical errors.
+    noteCliError(err, { handled: false })
     return super.catch(err)
   }
+
+  protected async finally (err: Error | undefined): Promise<any> {
+    // Here and not in catch(), so it also sends the errors a command printed
+    // itself and only exited on (see noteCliError).
+    await reportNotedCliError(() => ({ version: this.config.version, command: this.id, cliMode: detectCliMode() }))
+    return super.finally(err)
+  }
 }

packages/cli/src/commands/deploy.ts

@@ -47,6 +47,7 @@ import { uploadSnapshots } from '../services/snapshot-service.js'
 import { BrowserCheckBundle } from '../constructs/browser-check-bundle.js'
 import { Runtime } from '../runtimes/index.js'
 import { Bundler } from '../services/check-parser/bundler.js'
+import { noteCliError } from '../services/error-reporting/report.js'
 
 /**
  * The deploy payload fields that arrived with the preview endpoint, which a
@@ -122,6 +123,7 @@ function writeBackAlternatives (diff: DiffEntry[], project: Project, command: De
           command.logToStderr(skipped)
         }
         // The error names the files already rewritten, if any.
+        noteCliError(err)
         command.style.longError('Could not update your code.', err.message)
         command.logToStderr('Nothing was deployed.')
         command.exit(1)
@@ -987,6 +989,7 @@ export default class Deploy extends AuthCommand {
             : 'Re-run the command. While plans are switched off it deploys without one.',
         )
       } else if (err instanceof ProjectDeployCancelledError) {
+        noteCliError(err)
         this.style.longError('Your deployment was cancelled.', err.message)
       } else if (err instanceof ConflictError) {
         // deploy() waits-and-retries behind an in-progress deployment, so a 409

packages/cli/src/commands/destroy.ts

@@ -8,6 +8,7 @@ import { forceFlag } from '../helpers/flags.js'
 import { ProjectDeployCancelledError } from '../rest/projects.js'
 import { ConflictError } from '../rest/errors.js'
 import { prompt } from '../helpers/prompts.js'
+import { noteCliError } from '../services/error-reporting/report.js'
 
 export default class Destroy extends AuthCommand {
   static hidden = false
@@ -98,6 +99,7 @@ export default class Destroy extends AuthCommand {
     } catch (err: any) {
       this.style.actionFailure()
       if (err instanceof ProjectDeployCancelledError) {
+        noteCliError(err)
         this.style.longError('Your project could not be destroyed.', err.message)
       } else if (err instanceof ConflictError) {
         // deleteProject waits-and-retries behind an in-progress operation, so a

packages/cli/src/commands/import/plan.ts

@@ -38,6 +38,7 @@ import { ConstructExport, Session } from '../../constructs/session.js'
 import { Diagnostics } from '../../constructs/index.js'
 import { Runtime } from '../../runtimes/index.js'
 import { prompt } from '../../helpers/prompts.js'
+import { noteCliError } from '../../services/error-reporting/report.js'
 
 type FriendExports = {
   [type in FriendResourceSync['type']]: Map<string, ConstructExport>
@@ -798,6 +799,7 @@ ${chalk.cyan('For safety, resources are not deletable until the plan has been co
           )
         } catch (err) {
           if (err instanceof Error) {
+            noteCliError(err)
             this.style.longError(`Failed to install dependencies`, err.message) // TODO :
           }
 

packages/cli/src/commands/skills/install.ts

@@ -12,6 +12,7 @@ import {
   promptForPlatformTarget,
 } from '../../services/skills.js'
 import { prompt } from '../../helpers/prompts.js'
+import { noteCliError } from '../../services/error-reporting/report.js'
 
 const SKILL_FILENAME = 'SKILL.md'
 
@@ -82,6 +83,7 @@ export default class SkillsInstall extends BaseCommand {
     try {
       return await readSkillFile()
     } catch (err: any) {
+      noteCliError(err)
       this.error(err.message, { exit: 1 })
     }
   }
@@ -125,6 +127,7 @@ export default class SkillsInstall extends BaseCommand {
       const writtenPath = await writeSkillToTarget(targetDir, content)
       this.style.shortSuccess(`Installed Checkly agent skill to: ${writtenPath}`)
     } catch (err: any) {
+      noteCliError(err)
       this.error(err.message, { exit: 1 })
     }
   }

packages/cli/src/commands/test-sessions/list.ts

@@ -9,6 +9,7 @@ import {
   formatTestSessionsListNavigationHints,
   formatTestSessionsListPaginationInfo,
 } from '../../formatters/test-sessions.js'
+import { noteCliError } from '../../services/error-reporting/report.js'
 
 const statusOptions = ['running', 'failed', 'passed', 'cancelled'] as const
 const providerOptions = ['github', 'vercel', 'api', 'trigger', 'pw_reporter'] as const
@@ -154,6 +155,7 @@ export default class TestSessionsList extends AuthCommand {
       from = parseUnixTimestamp(flags.from, 'from')
       to = parseUnixTimestamp(flags.to, 'to')
     } catch (err: any) {
+      noteCliError(err)
       this.error(err.message)
     }
 

packages/cli/src/helpers/command-style.ts

@@ -4,6 +4,7 @@ import { ux } from '@oclif/core'
 import { BaseCommand } from '../commands/baseCommand.js'
 import { Diagnostics } from '../constructs/diagnostics.js'
 import { NotFoundError } from '../rest/errors.js'
+import { noteCliError } from '../services/error-reporting/report.js'
 import { wrap } from './wrap.js'
 import logSymbols from 'log-symbols'
 
@@ -98,6 +99,9 @@ export class CommandStyle {
   }
 
   longError (title: string, message: string | Error) {
+    // Commands print the errors they catch here, so a CLI bug among them is
+    // reported when the command ends.
+    noteCliError(message)
     const hint = message instanceof NotFoundError ? this.c.notFoundHint() : undefined
     if (this.outputFormat === 'json') {
       // The searched account as data, so an agent need not parse the hint.

packages/cli/src/services/abstract-check-runner.ts

@@ -11,6 +11,7 @@ import {
   TestSessionSchedulingOperation,
 } from '../rest/test-sessions.js'
 import { PlaywrightCheck } from '../constructs/index.js'
+import { noteCliError } from './error-reporting/report.js'
 
 // eslint-disable-next-line no-restricted-syntax
 export enum Events {
@@ -234,6 +235,9 @@ export default abstract class AbstractCheckRunner extends EventEmitter {
       this.emit(Events.RUN_FINISHED, testSessionId)
     } catch (err) {
       this.disableAllTimeouts()
+      // Commands print this error and exit without throwing it, so a CLI bug
+      // in it is noted here to be reported when the command ends.
+      noteCliError(err)
       this.emit(Events.ERROR, err)
     } finally {
       // Stop any in-flight scheduling long-poll so it doesn't hold the