diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 90d2d66c..abcffeab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,10 +89,27 @@ jobs: - run: pnpm install --frozen-lockfile - name: Add SHORT_SHA env property with commit short sha run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + # Release builds report crashes in the CLI's own code to Checkly (see + # src/services/error-reporting/). The DSN is public by nature: it can only + # send events to one project. Builds without the file report nothing. + - name: Write the error reporting DSN for 'cli' package + if: ${{ vars.CHECKLY_CLI_ERROR_DSN != '' }} + env: + DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }} + run: | + jq -n --arg dsn "$DSN" '{dsn: $dsn}' > error-reporting.json + working-directory: packages/cli - name: Set version and publish prerelease for 'cli' package + env: + DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }} run: | pnpm version ${{ github.event.release.tag_name }}-prerelease-${{ env.SHORT_SHA }} --no-git-tag-version pnpm pack + # A package without the file would silently report nothing. + if [ -n "$DSN" ] && ! tar -tzf checkly-*.tgz | grep -qx 'package/error-reporting.json'; then + echo "::error::error-reporting.json is missing from the packed 'cli' package" + exit 1 + fi npm publish checkly-*.tgz --provenance --tag prerelease working-directory: packages/cli - name: Set version and publish prerelease for 'create-cli' package @@ -158,10 +175,27 @@ jobs: # Ensure that the README is published with the package - run: rm -f packages/cli/README.md && cp README.md packages/cli - run: pnpm install --frozen-lockfile + # Release builds report crashes in the CLI's own code to Checkly (see + # src/services/error-reporting/). The DSN is public by nature: it can only + # send events to one project. Builds without the file report nothing. + - name: Write the error reporting DSN for 'cli' package + if: ${{ vars.CHECKLY_CLI_ERROR_DSN != '' }} + env: + DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }} + run: | + jq -n --arg dsn "$DSN" '{dsn: $dsn}' > error-reporting.json + working-directory: packages/cli - name: Set version and publish 'cli' package + env: + DSN: ${{ vars.CHECKLY_CLI_ERROR_DSN }} run: | pnpm version ${{ github.event.release.tag_name }} --no-git-tag-version pnpm pack + # A package without the file would silently report nothing. + if [ -n "$DSN" ] && ! tar -tzf checkly-*.tgz | grep -qx 'package/error-reporting.json'; then + echo "::error::error-reporting.json is missing from the packed 'cli' package" + exit 1 + fi npm publish checkly-*.tgz --provenance --tag ${{ needs.validate-tag.outputs.dist-tag }} working-directory: packages/cli - name: Set version and publish 'create-cli' package diff --git a/.gitignore b/.gitignore index 84a85a6a..34622d06 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ local **/checkly-github-report.md **/checkly-summary.md **/e2e/__tests__/fixtures/empty-project/e2e-test-project-* + +# Written by the release workflow; see packages/cli/src/services/error-reporting/report.ts +packages/cli/error-reporting.json diff --git a/CLAUDE.md b/CLAUDE.md index 1a5fad80..556ad51d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,6 +121,7 @@ Source in `src/ai-context/`, built during `prepare`. Generates examples from fix - `CHECKLY_CLI_MODE` — force the CLI mode (`interactive`, `agent` or `ci`) instead of detecting it; an explicit `interactive` also lets `checkly login` show its code without a detected terminal (e2e harness, terminals Node doesn't recognise). The mode also sets the `x-checkly-source` header: `agent` reports `AGENT`, which changes how the backend classifies the requests (see `requestInterceptor` in `src/rest/api.ts`) - `CHECKLY_NO_BROWSER` — set to `1` so `checkly login` prints the login URL without trying to open a browser - `CHECKLY_NO_DELEGATION` — set to `1` to stop the CLI from handing every command to the project's own `node_modules/checkly` (searched upwards from the `--config`/`-c` file's directory, or the working directory) when that is a different install (`bin/launch.cjs`, `src/services/project-cli.ts`). Useful when running a dev checkout in a project that has its own `checkly`; constructs then fail to register unless the project resolves `checkly` to the same install +- `CHECKLY_NO_ERROR_REPORTING` — set to `1` (or set `DO_NOT_TRACK=1`) to stop the CLI from reporting crashes in its own code to Checkly. Only release builds report: the release workflow writes the DSN from the `CHECKLY_CLI_ERROR_DSN` repository variable to `packages/cli/error-reporting.json`. Only TypeError, ReferenceError and RangeError raised in the CLI's own files are sent (also when wrapped as a `cause`, or printed by a command that catches it: `BaseCommand.finally` sends what `noteCliError` kept), once per run, without paths, credentials, arguments or environment (`src/services/error-reporting/`) - `CHECKLY_CACHE_DIR` — override the CLI's cache directory (embedded-package tarball downloads) - `CHECKLY_SKIP_NODE_VERSION_CHECK` — set to `1` to bypass the bin's hard Node version preflight (unsupported Node may then fail in unexpected ways); must be set in the shell environment — the preflight runs before `.env` is loaded - `CHECKLY_LOCKFILE_PRUNE` — set to `0` to disable pruning the bundled lockfile to the code bundle's contents; when a lockfile is bundled this also disables `bundle.packages.prune` (the manifest rewrite rolls back) diff --git a/README.md b/README.md index 7677d25d..5fda1ee4 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,34 @@ This provides your AI with documentation, best practices, and examples so it can Check out our [AI Agents & Coding Assistants docs](https://www.checklyhq.com/docs/integrations/ai/overview) for more best practices and examples. +# Error reporting + +When the CLI crashes because of a bug in its own code, it sends one report about the crash to Checkly, so we can find +and fix the bug. Only crashes inside the CLI's own files are reported (a `TypeError`, `ReferenceError` or +`RangeError`). Errors in your own code, failed checks, API errors and network problems are never reported. Invalid +configuration is normally shown as a clear message and not reported; if it makes the CLI crash instead, that crash is +reported, because the CLI should have caught the mistake. + +A report contains: + +- the error type, its message and the stack trace; paths in the CLI's own files are shortened to the file inside the + CLI (for example `checkly/dist/commands/deploy.js`), paths in the CLI's dependencies start at `node_modules/`, and + every other path is replaced with a placeholder +- the CLI version, the command name (for example `deploy`), and whether the CLI runs in a terminal, in CI or for an + AI agent +- your operating system, CPU architecture and Node.js version + +A report never contains your API key, account ID, command arguments, environment variables, computer name or the +contents of your files. Credentials found in an error message are removed before sending. + +To turn error reporting off, set either environment variable: + +```bash +export CHECKLY_NO_ERROR_REPORTING=1 +# or the cross-tool convention +export DO_NOT_TRACK=1 +``` + ## Need help? diff --git a/packages/cli/package.json b/packages/cli/package.json index f36e2821..20b97244 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -43,7 +43,8 @@ "/assets", "/constructs.js", "/constructs.d.ts", - "/oclif.manifest.json" + "/oclif.manifest.json", + "/error-reporting.json" ], "oclif": { "bin": "checkly", diff --git a/packages/cli/src/commands/__tests__/base-command-error-reporting.spec.ts b/packages/cli/src/commands/__tests__/base-command-error-reporting.spec.ts new file mode 100644 index 00000000..f23caaab --- /dev/null +++ b/packages/cli/src/commands/__tests__/base-command-error-reporting.spec.ts @@ -0,0 +1,99 @@ +import http from 'node:http' +import { Config } from '@oclif/core' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +import { BaseCommand } from '../baseCommand.js' +import { PACKAGE_ROOT } from '../../services/error-reporting/classify.js' +import { resetErrorReporting, setDefaultErrorReportOptions } from '../../services/error-reporting/report.js' + +// Both commands live in this file, which is inside the CLI's package, so their +// TypeErrors count as bugs in the CLI's own code. + +class CrashingCommand extends BaseCommand { + run (): Promise { + const resource: any = undefined + return resource.logicalId + } +} + +class PrintingCommand extends BaseCommand { + run (): Promise { + try { + const resource: any = undefined + return resource.logicalId + } catch (err) { + this.style.longError('Could not read the resource.', err as Error) + this.exit(1) + } + } +} + +interface Envelope { level: string, exception: { values: Array<{ type: string, mechanism: { handled: boolean } }> } } + +let config: Config +let server: http.Server +let dsn: string +let received: Envelope[] + +beforeAll(async () => { + config = await Config.load(PACKAGE_ROOT) + server = http.createServer((req, res) => { + let body = '' + req.on('data', chunk => { + body += chunk + }) + req.on('end', () => { + received.push(JSON.parse(body.split('\n')[2])) + res.writeHead(200).end('{}') + }) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as { port: number } + dsn = `http://${'a'.repeat(72)}@127.0.0.1:${port}/123` +}) + +afterAll(async () => { + await new Promise(resolve => server.close(resolve)) +}) + +beforeEach(() => { + received = [] + resetErrorReporting() + setDefaultErrorReportOptions({ dsn, env: {} }) + // A release version: development builds never report. + config.version = '7.1.0' + // Keeps init from asking the npm registry for the latest version. + vi.stubEnv('CHECKLY_E2E_CLI_VERSION', '7.1.0') + vi.spyOn(process.stderr, 'write').mockReturnValue(true) +}) + +afterEach(() => { + resetErrorReporting() + vi.unstubAllEnvs() + vi.restoreAllMocks() +}) + +describe('BaseCommand error reporting', () => { + it('reports a CLI bug that ends a command as an unhandled crash, before the command fails', async () => { + await expect(new CrashingCommand([], config)._run()).rejects.toThrow(TypeError) + + expect(received).toHaveLength(1) + expect(received[0].level).toBe('fatal') + expect(received[0].exception.values[0]).toMatchObject({ type: 'TypeError', mechanism: { handled: false } }) + }) + + it('reports a CLI bug a command printed with longError and exited on', async () => { + await expect(new PrintingCommand([], config)._run()).rejects.toThrow('EEXIT: 1') + + expect(received).toHaveLength(1) + expect(received[0].level).toBe('error') + expect(received[0].exception.values[0]).toMatchObject({ type: 'TypeError', mechanism: { handled: true } }) + }) + + it('sends nothing for a development build', async () => { + config.version = '0.0.1-dev' + await expect(new CrashingCommand([], config)._run()).rejects.toThrow(TypeError) + + expect(received).toHaveLength(0) + }) +}) diff --git a/packages/cli/src/commands/__tests__/confirm-flow-deploy.spec.ts b/packages/cli/src/commands/__tests__/confirm-flow-deploy.spec.ts index 57ad7f60..e379d03b 100644 --- a/packages/cli/src/commands/__tests__/confirm-flow-deploy.spec.ts +++ b/packages/cli/src/commands/__tests__/confirm-flow-deploy.spec.ts @@ -1534,7 +1534,9 @@ new ApiCheck('api', { await expect(Deploy.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_1') - expect(ctx.style.longError).toHaveBeenCalledWith('Could not update your code.', 'Could not write api.check.ts: EACCES. No file was changed.') + expect(ctx.style.longError).toHaveBeenCalledWith('Could not update your code.', expect.objectContaining({ + message: 'Could not write api.check.ts: EACCES. No file was changed.', + })) expect(ctx.logged.join('\n')).not.toContain('Updated') expect(api.projects.deploy).not.toHaveBeenCalled() }) diff --git a/packages/cli/src/commands/baseCommand.ts b/packages/cli/src/commands/baseCommand.ts index 7974c8dc..0c34b9ae 100644 --- a/packages/cli/src/commands/baseCommand.ts +++ b/packages/cli/src/commands/baseCommand.ts @@ -12,6 +12,8 @@ import { findStaleSkills } from '../services/skills.js' import { PackageJsonFile } from '../services/check-parser/package-files/package-json-file.js' import { detectNearestPackageJson } from '../services/check-parser/package-files/package-manager.js' import { isDevelopmentBuild } from '../services/project-cli.js' +import { noteCliError, reportNotedCliError } from '../services/error-reporting/report.js' +import { detectCliMode } from '../helpers/cli-mode.js' const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -195,7 +197,14 @@ export abstract class BaseCommand extends Command { if (err instanceof NotFoundError) { err.message = this.withNotFoundHint(err.message) } - // TODO: we can add Sentry here and log critical errors. + noteCliError(err, { handled: false }) return super.catch(err) } + + protected async finally (err: Error | undefined): Promise { + // Here and not in catch(), so it also sends the errors a command printed + // itself and only exited on (see noteCliError). + await reportNotedCliError(() => ({ version: this.config.version, command: this.id, cliMode: detectCliMode() })) + return super.finally(err) + } } diff --git a/packages/cli/src/commands/deploy.ts b/packages/cli/src/commands/deploy.ts index 314ac2ca..c88da42c 100644 --- a/packages/cli/src/commands/deploy.ts +++ b/packages/cli/src/commands/deploy.ts @@ -122,7 +122,7 @@ function writeBackAlternatives (diff: DiffEntry[], project: Project, command: De command.logToStderr(skipped) } // The error names the files already rewritten, if any. - command.style.longError('Could not update your code.', err.message) + command.style.longError('Could not update your code.', err) command.logToStderr('Nothing was deployed.') command.exit(1) } @@ -987,7 +987,7 @@ export default class Deploy extends AuthCommand { : 'Re-run the command. While plans are switched off it deploys without one.', ) } else if (err instanceof ProjectDeployCancelledError) { - this.style.longError('Your deployment was cancelled.', err.message) + this.style.longError('Your deployment was cancelled.', err) } else if (err instanceof ConflictError) { // deploy() waits-and-retries behind an in-progress deployment, so a 409 // only reaches here once that wait exceeded its deadline. diff --git a/packages/cli/src/commands/destroy.ts b/packages/cli/src/commands/destroy.ts index d1e7713b..5a85ade7 100644 --- a/packages/cli/src/commands/destroy.ts +++ b/packages/cli/src/commands/destroy.ts @@ -98,7 +98,7 @@ export default class Destroy extends AuthCommand { } catch (err: any) { this.style.actionFailure() if (err instanceof ProjectDeployCancelledError) { - this.style.longError('Your project could not be destroyed.', err.message) + this.style.longError('Your project could not be destroyed.', err) } else if (err instanceof ConflictError) { // deleteProject waits-and-retries behind an in-progress operation, so a // 409 only reaches here once that wait exceeded its deadline. diff --git a/packages/cli/src/commands/import/plan.ts b/packages/cli/src/commands/import/plan.ts index 2706d295..4d82aa4a 100644 --- a/packages/cli/src/commands/import/plan.ts +++ b/packages/cli/src/commands/import/plan.ts @@ -798,7 +798,7 @@ ${chalk.cyan('For safety, resources are not deletable until the plan has been co ) } catch (err) { if (err instanceof Error) { - this.style.longError(`Failed to install dependencies`, err.message) // TODO : + this.style.longError(`Failed to install dependencies`, err) // TODO : } this.style.comment( diff --git a/packages/cli/src/commands/skills/install.ts b/packages/cli/src/commands/skills/install.ts index 94549547..9832a94e 100644 --- a/packages/cli/src/commands/skills/install.ts +++ b/packages/cli/src/commands/skills/install.ts @@ -12,6 +12,7 @@ import { promptForPlatformTarget, } from '../../services/skills.js' import { prompt } from '../../helpers/prompts.js' +import { noteCliError } from '../../services/error-reporting/report.js' const SKILL_FILENAME = 'SKILL.md' @@ -82,6 +83,7 @@ export default class SkillsInstall extends BaseCommand { try { return await readSkillFile() } catch (err: any) { + noteCliError(err) this.error(err.message, { exit: 1 }) } } @@ -125,6 +127,7 @@ export default class SkillsInstall extends BaseCommand { const writtenPath = await writeSkillToTarget(targetDir, content) this.style.shortSuccess(`Installed Checkly agent skill to: ${writtenPath}`) } catch (err: any) { + noteCliError(err) this.error(err.message, { exit: 1 }) } } diff --git a/packages/cli/src/helpers/command-style.ts b/packages/cli/src/helpers/command-style.ts index 438d8cce..18f14509 100644 --- a/packages/cli/src/helpers/command-style.ts +++ b/packages/cli/src/helpers/command-style.ts @@ -4,6 +4,7 @@ import { ux } from '@oclif/core' import { BaseCommand } from '../commands/baseCommand.js' import { Diagnostics } from '../constructs/diagnostics.js' import { NotFoundError } from '../rest/errors.js' +import { noteCliError } from '../services/error-reporting/report.js' import { wrap } from './wrap.js' import logSymbols from 'log-symbols' @@ -98,6 +99,9 @@ export class CommandStyle { } longError (title: string, message: string | Error) { + // Commands print the errors they catch here, so a CLI bug among them is + // reported when the command ends. + noteCliError(message) const hint = message instanceof NotFoundError ? this.c.notFoundHint() : undefined if (this.outputFormat === 'json') { // The searched account as data, so an agent need not parse the hint. diff --git a/packages/cli/src/services/__tests__/abstract-check-runner.spec.ts b/packages/cli/src/services/__tests__/abstract-check-runner.spec.ts index 74e5a2cc..11312807 100644 --- a/packages/cli/src/services/__tests__/abstract-check-runner.spec.ts +++ b/packages/cli/src/services/__tests__/abstract-check-runner.spec.ts @@ -34,6 +34,10 @@ vi.mock('../socket-client.js', () => ({ }, })) +vi.mock('../error-reporting/report.js', () => ({ + noteCliError: vi.fn(), +})) + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -42,6 +46,7 @@ import { SocketClient } from '../socket-client.js' import { assets, testSessions } from '../../rest/api.js' import { PlaywrightCheck } from '../../constructs/index.js' import { TestSessionSchedulingFailedError } from '../../rest/test-sessions.js' +import { noteCliError } from '../error-reporting/report.js' /** Minimal concrete subclass — scheduleChecks immediately returns with zero checks so the runner exits cleanly. */ class StubCheckRunner extends AbstractCheckRunner { @@ -565,6 +570,68 @@ describe('AbstractCheckRunner — timeout racing an in-flight result', () => { expect(events.filter(e => e === `${Events.CHECK_FINISHED}:api`)).toHaveLength(1) }) + it('fails the run with an error a listener throws while a result is processed', async () => { + const runner = new TwoCheckRunner('acc-1', DEFAULT_CHECK_RUN_TIMEOUT_SECONDS, false) + const reporterBug = new TypeError('Cannot read properties of undefined') + runner.on(Events.CHECK_SUCCESSFUL, () => { + throw reporterBug + }) + const errors: unknown[] = [] + runner.on(Events.ERROR, err => errors.push(err)) + + const run = runner.run() + await flush() + messageHandler!(resultTopic('seq-api'), JSON.stringify({ + result: { hasFailures: false }, + testResultId: 'tr-1', + resultType: 'FINAL', + })) + await flush() + await run + + expect(errors).toEqual([reporterBug]) + // Commands print the error without throwing it, so it is noted for the report here. + expect(noteCliError).toHaveBeenCalledWith(reporterBug) + }) + + it.each([ + // A reporter's onCheckEnd for a check that timed out runs in the timeout's timer. + [Events.CHECK_FINISHED, (DEFAULT_CHECK_RUN_TIMEOUT_SECONDS + MAX_SCHEDULING_DELAY_SECONDS) * 1000], + // The hint that checks are still being scheduled is emitted from a timer after 20 s. + [Events.MAX_SCHEDULING_DELAY_EXCEEDED, 20_000], + ])('fails the run with an error a listener to %s throws from a timer', async (event, delayMs) => { + const runner = new TwoCheckRunner('acc-1', DEFAULT_CHECK_RUN_TIMEOUT_SECONDS, false) + const reporterBug = new TypeError('Cannot read properties of undefined') + runner.on(event, () => { + throw reporterBug + }) + const errors: unknown[] = [] + runner.on(Events.ERROR, err => errors.push(err)) + + const run = runner.run() + await flush() + await vi.advanceTimersByTimeAsync(delayMs) + await run + + expect(errors).toEqual([reporterBug]) + expect(noteCliError).toHaveBeenCalledWith(reporterBug) + }) + + it('fails the run when a result message cannot be read', async () => { + const runner = new TwoCheckRunner('acc-1', DEFAULT_CHECK_RUN_TIMEOUT_SECONDS, false) + const errors: unknown[] = [] + runner.on(Events.ERROR, err => errors.push(err)) + + const run = runner.run() + await flush() + expect(() => messageHandler!(resultTopic('seq-api'), '{not json')).not.toThrow() + await flush() + await run + + expect(errors).toHaveLength(1) + expect(errors[0]).toBeInstanceOf(SyntaxError) + }) + it('restarts the timeout before fetching an ATTEMPT result, so a fetch near the deadline does not time out', async () => { const runner = new TwoCheckRunner('acc-1', DEFAULT_CHECK_RUN_TIMEOUT_SECONDS, false) record(runner) diff --git a/packages/cli/src/services/abstract-check-runner.ts b/packages/cli/src/services/abstract-check-runner.ts index 9789306a..2d7dbca9 100644 --- a/packages/cli/src/services/abstract-check-runner.ts +++ b/packages/cli/src/services/abstract-check-runner.ts @@ -11,6 +11,7 @@ import { TestSessionSchedulingOperation, } from '../rest/test-sessions.js' import { PlaywrightCheck } from '../constructs/index.js' +import { noteCliError } from './error-reporting/report.js' // eslint-disable-next-line no-restricted-syntax export enum Events { @@ -91,6 +92,10 @@ export default abstract class AbstractCheckRunner extends EventEmitter { readonly maxFailures?: number // Set when the run stopped early; results that arrive afterwards are dropped. private _maxFailuresReached = false + // Rejects the run with an error thrown while processing a result, e.g. by a + // reporter. Nothing awaits the queued tasks, so without it such an error + // would end the process as an unhandled rejection. + private failProcessing?: (err: unknown) => void constructor ( accountId: string, @@ -156,6 +161,12 @@ export default abstract class AbstractCheckRunner extends EventEmitter { return } + const processingFailed = new Promise((_, reject) => { + this.failProcessing = reject + }) + // Observed by the race below; keeps a failure before it from becoming an unhandled rejection. + processingFailed.catch(() => {}) + socketClient = await SocketClient.connect() // Configure the socket listener and allChecksFinished listener before starting checks to avoid race conditions @@ -227,13 +238,16 @@ export default abstract class AbstractCheckRunner extends EventEmitter { // Start the queue after the test session run rest call is completed to avoid race conditions this.queue.start() - await Promise.race([allChecksFinished, maxFailuresReached, schedulingFailed]) + await Promise.race([allChecksFinished, maxFailuresReached, schedulingFailed, processingFailed]) if (this.maxFailuresReached) { this.emit(Events.MAX_FAILURES_REACHED, this.maxFailures) } this.emit(Events.RUN_FINISHED, testSessionId) } catch (err) { this.disableAllTimeouts() + // Commands print this error and exit without throwing it, so a CLI bug + // in it is noted here to be reported when the command ends. + noteCliError(err) this.emit(Events.ERROR, err) } finally { // Stop any in-flight scheduling long-poll so it doesn't hold the @@ -293,14 +307,19 @@ export default abstract class AbstractCheckRunner extends EventEmitter { private async configureResultListener (checkRunSuiteId: string, socketClient: MqttClient): Promise { socketClient.on('message', (topic: string, rawMessage: string | Buffer) => { - const message = JSON.parse(rawMessage.toString('utf8')) - const topicComponents = topic.split('/') - const sequenceId = topicComponents[4] - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const checkRunId = topicComponents[5] - const subtopic = topicComponents[6] - - this.queue.add(() => this.processMessage(sequenceId, subtopic, message)) + try { + const message = JSON.parse(rawMessage.toString('utf8')) + const topicComponents = topic.split('/') + const sequenceId = topicComponents[4] + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const checkRunId = topicComponents[5] + const subtopic = topicComponents[6] + + this.queue.add(() => this.processMessage(sequenceId, subtopic, message)) + .catch(err => this.failProcessing?.(err)) + } catch (err) { + this.failProcessing?.(err) + } }) await socketClient.subscribeAsync(`account/${this.accountId}/ad-hoc-check-results/${checkRunSuiteId}/+/+/+`) } @@ -483,7 +502,7 @@ export default abstract class AbstractCheckRunner extends EventEmitter { ? DEFAULT_PLAYWRIGHT_CHECK_RUN_TIMEOUT_SECONDS : this.timeout const checkTimeout = started ? executionTimeout : executionTimeout + MAX_SCHEDULING_DELAY_SECONDS - this.timeouts.set(sequenceId, setTimeout(() => { + this.timeouts.set(sequenceId, setTimeout(this.failingRun(() => { this.timeouts.delete(sequenceId) let errorMessage = started ? `Reached timeout of ${checkTimeout} seconds waiting for check result.` @@ -501,7 +520,7 @@ export default abstract class AbstractCheckRunner extends EventEmitter { } this.emit(Events.CHECK_FAILED, sequenceId, check, errorMessage, { timedOut: true }) this.emit(Events.CHECK_FINISHED, check) - }, checkTimeout * 1000)) + }), checkTimeout * 1000)) } private disableAllTimeouts () { @@ -523,10 +542,10 @@ export default abstract class AbstractCheckRunner extends EventEmitter { return } this.schedulingDelayExceededTimeout = setTimeout( - () => { + this.failingRun(() => { this.emit(Events.MAX_SCHEDULING_DELAY_EXCEEDED) this.schedulingDelayExceededTimeout = undefined - }, + }), DEFAULT_SCHEDULING_DELAY_EXCEEDED_MS, ) this.on(Events.CHECK_INPROGRESS, () => { @@ -535,6 +554,21 @@ export default abstract class AbstractCheckRunner extends EventEmitter { }) } + /** + * `fn` for a timer: an error it throws, e.g. from a reporter listening to + * the events it emits, fails the run instead of ending the process as an + * uncaught exception. + */ + private failingRun (fn: () => void): () => void { + return () => { + try { + fn() + } catch (err) { + this.failProcessing?.(err) + } + } + } + private disableTimeout (timeoutKey: string) { const timeout = this.timeouts.get(timeoutKey) clearTimeout(timeout) diff --git a/packages/cli/src/services/error-reporting/__tests__/error-reporting.spec.ts b/packages/cli/src/services/error-reporting/__tests__/error-reporting.spec.ts new file mode 100644 index 00000000..a90205b1 --- /dev/null +++ b/packages/cli/src/services/error-reporting/__tests__/error-reporting.spec.ts @@ -0,0 +1,443 @@ +import fs from 'node:fs/promises' +import http from 'node:http' +import os from 'node:os' +import path from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { findCliBug, isCliBug } from '../classify.js' +import { buildErrorEvent, releaseEnvironment, type ErrorReportContext } from '../event.js' +import { + isErrorReportingDisabled, + noteCliError, + reportNotedCliError, + resetErrorReporting, + type ErrorReportOptions, +} from '../report.js' +import { scrubMessage } from '../scrub.js' + +const KEY = 'a'.repeat(72) + +let installDir: string +let cliBug: Error +let dependencyBug: Error +let invalidUrl: Error +let privatePackageBug: Error + +async function errorThrownIn (file: string, code: string): Promise { + await fs.mkdir(path.dirname(file), { recursive: true }) + await fs.writeFile(file, `export function run () { ${code} }\n`) + const { run } = await import(pathToFileURL(file).href) + try { + run() + } catch (err) { + return err as Error + } + throw new Error(`${file} did not throw`) +} + +/** Notes `err` and sends the report at once, as a command that ends with it does. */ +async function report (err: unknown, context: ErrorReportContext, options: ErrorReportOptions): Promise { + noteCliError(err, { handled: !!context.handled }) + await reportNotedCliError(() => context, options) +} + +interface Received { url: string, body: string } + +async function withIngest (fn: (dsn: string, received: Received[]) => Promise): Promise { + const received: Received[] = [] + const server = http.createServer((req, res) => { + let body = '' + req.on('data', chunk => { + body += chunk + }) + req.on('end', () => { + received.push({ url: req.url ?? '', body }) + res.writeHead(200, { 'Content-Type': 'application/json' }).end('{}') + }) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as { port: number } + try { + await fn(`http://${KEY}@127.0.0.1:${port}/123`, received) + } finally { + await new Promise(resolve => server.close(resolve)) + } +} + +beforeAll(async () => { + // The space makes ES module stack frames carry %20 in their file URLs. + installDir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'checkly error reporting-'))) + await fs.writeFile(path.join(installDir, 'package.json'), JSON.stringify({ dependencies: { 'some-dep': '1.0.0' } })) + cliBug = await errorThrownIn( + path.join(installDir, 'dist', 'commands', 'deploy.mjs'), + 'const resource = undefined; return resource.logicalId', + ) + dependencyBug = await errorThrownIn( + path.join(installDir, 'node_modules', 'some-dep', 'index.mjs'), + 'null.x', + ) + privatePackageBug = await errorThrownIn( + path.join(installDir, 'node_modules', '@acme-internal', 'billing', 'index.mjs'), + 'null.x', + ) + invalidUrl = await errorThrownIn( + path.join(installDir, 'dist', 'services', 'proxy.mjs'), + 'new URL(\'not a url\')', + ) +}) + +afterAll(async () => { + await fs.rm(installDir, { recursive: true, force: true }) +}) + +beforeEach(() => { + resetErrorReporting() +}) + +describe('isCliBug', () => { + it('accepts a runtime error raised in the CLI\'s own files', () => { + expect(cliBug).toBeInstanceOf(TypeError) + expect(isCliBug(cliBug, installDir)).toBe(true) + }) + + it('accepts it when the stack names the file by URL, as the installed CLI\'s ES modules do', () => { + // Vitest loads modules itself and its stacks show plain paths, so the URL form is built here: + // file:///C:/... on Windows, with the space in the install path as %20. + const url = pathToFileURL(path.join(installDir, 'dist', 'commands', 'deploy.js')).href + expect(url).toContain('%20') + const err = new TypeError('Cannot read properties of undefined (reading \'logicalId\')') + err.stack = `TypeError: ${err.message}\n at Deploy.run (${url}:12:34)\n at async Deploy._run (${url}:1:1)` + expect(isCliBug(err, installDir)).toBe(true) + }) + + it('accepts it when the CLI\'s code ran through eval, whose frames name the file differently', () => { + const file = path.join(installDir, 'dist', 'services', 'loader.js') + const err = new TypeError('Cannot read properties of undefined (reading \'x\')') + err.stack = `TypeError: ${err.message}\n` + + ` at eval (eval at load (${file}:1:2), :1:1)\n` + + ` at load (${file}:1:2)` + expect(isCliBug(err, installDir)).toBe(true) + }) + + it.each([ + ['String.repeat', '\'x\'.repeat(-1)'], + ['Array.reduce', '[].reduce((a, b) => a)'], + ['Date.toISOString', 'new Date(\'nope\').toISOString()'], + ['Array.map', '[1].map(undefined)'], + ['JSON.stringify', 'const a = {}; a.self = a; JSON.stringify(a)'], + ])('accepts an error that %s throws when the CLI\'s code calls it wrongly', async (name, code) => { + const file = path.join(installDir, 'dist', 'builtins', `${name}.mjs`) + const err = await errorThrownIn(file, code) + // The innermost frame is the built-in's, not a file's. + // The message may span lines (JSON.stringify's does), so look for the first frame. + expect(err.stack?.split('\n').find(line => line.startsWith(' at '))).toMatch(/\((?:|native)\)$/) + expect(isCliBug(err, installDir)).toBe(true) + }) + + it('rejects the same error when it was raised outside the CLI, e.g. in a user\'s check file', async () => { + const elsewhere = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-other-')) + try { + expect(isCliBug(cliBug, elsewhere)).toBe(false) + } finally { + await fs.rm(elsewhere, { recursive: true, force: true }) + } + }) + + it('rejects errors raised in a dependency of the CLI', () => { + expect(dependencyBug).toBeInstanceOf(TypeError) + expect(isCliBug(dependencyBug, installDir)).toBe(false) + }) + + it('rejects the TypeError Node raises for a malformed URL the user gave', () => { + expect(invalidUrl).toBeInstanceOf(TypeError) + expect((invalidUrl as NodeJS.ErrnoException).code).toBe('ERR_INVALID_URL') + // The same place counts as a CLI bug for any other TypeError, so only the code rejects it. + const other = new TypeError('x') + other.stack = invalidUrl.stack!.replace(/^TypeError.*$/m, 'TypeError: x') + expect(isCliBug(other, installDir)).toBe(true) + expect(isCliBug(invalidUrl, installDir)).toBe(false) + }) + + it('rejects errors the CLI throws on purpose', () => { + const intentional = new Error('Your Checkly configuration file is not valid.') + intentional.stack = cliBug.stack!.replace(/^TypeError: .*$/m, 'Error: Your Checkly configuration file is not valid.') + expect(isCliBug(intentional, installDir)).toBe(false) + expect(isCliBug('a string', installDir)).toBe(false) + }) +}) + +describe('findCliBug', () => { + it('finds the CLI bug a wrapper hides in its cause, as loading a check file does', () => { + const wrapper = new Error(`Error loading file '${installDir}/__checks__/api.check.ts'\n${cliBug.stack}`, { + cause: cliBug, + }) + expect(isCliBug(wrapper, installDir)).toBe(false) + expect(findCliBug(wrapper, installDir)).toBe(cliBug) + expect(findCliBug(new Error('wrapper', { cause: dependencyBug }), installDir)).toBeUndefined() + }) +}) + +describe('scrubMessage', () => { + const posix = { cwd: '/home/ann/acme', home: '/home/ann', packageRoot: '/opt/checkly', platform: 'linux' as const } + + it('removes credentials and paths that name the user', () => { + const env = { CHECKLY_API_KEY: 'my-secret-api-key-value' } + const message = [ + 'key cu_0123456789abcdef0123456789abcdef', + 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.e30.abc', + 'https://user:pass@proxy.example.com:8080', + 'token=abc123', + 'https://x/cb?access_token=SECRET1 client_secret=SECRET2 accessToken: SECRET3', + 'env my-secret-api-key-value', + `home ${os.homedir()}/file`, + `cwd ${process.cwd()}/checkly.config.ts`, + ].join('\n') + + const scrubbed = scrubMessage(message, env, { packageRoot: installDir }) + + expect(scrubbed).not.toContain('cu_0123') + expect(scrubbed).not.toContain('eyJhbGci') + expect(scrubbed).not.toContain('user:pass') + expect(scrubbed).not.toContain('abc123') + expect(scrubbed).not.toMatch(/SECRET[123]/) + expect(scrubbed).not.toContain('my-secret-api-key-value') + expect(scrubbed).not.toContain(os.homedir()) + expect(scrubbed).toMatch(/cwd $/) + }) + + it('removes whole passwords, also with an @ or, when quoted, a space in them', () => { + expect(scrubMessage('via http://bob:p@ss@proxy:3128 failed', {}, posix)) + .toBe('via http://@proxy:3128 failed') + // Not a valid URL, so it cannot be told where the password ends: all of it is withheld. + expect(scrubMessage('via http://bob:pa/ss@proxy:8080 failed', {}, posix)).toBe('via http:// failed') + expect(scrubMessage('GET https://admin:s3cret@api.example.com/v1/x?y=1 failed', {}, posix)) + .toBe('GET https://@api.example.com failed') + for (const char of ['\'', '"', '\x60', '<', '>']) { + expect(scrubMessage(`via http://bob:pa${char}ss@proxy:8080 failed`, {}, posix)) + .toBe('via http://@proxy:8080 failed') + } + // A quote around the URL stays outside it. + expect(scrubMessage('connect \'http://bob:pass@proxy:8080\' failed', {}, posix)) + .toBe('connect \'http://@proxy:8080\' failed') + // An @ in the path is not a credential. + expect(scrubMessage('see https://example.com/users/@ann for more', {}, posix)) + .toBe('see https://example.com/users/@ann for more') + expect(scrubMessage('{"password": "hunter two", "user": "bob"}', {}, posix)) + .toBe('{"password": "", "user": "bob"}') + expect(scrubMessage('password=\'hunter two\'', {}, posix)).toBe('password=\'\'') + expect(scrubMessage('secret: "unterminated', {}, posix)).toBe('secret: "') + }) + + it('removes whole paths only, so a short working directory leaves other text alone', () => { + const scrubbed = scrubMessage('Cannot read \'/app/__checks__/acme.check.ts\' as application/json from /app.', {}, { + cwd: '/app', + home: '/root', + packageRoot: '/usr/lib/node_modules/checkly', + platform: 'linux', + }) + expect(scrubbed).toBe('Cannot read \'\' as application/json from .') + }) + + it('names the CLI\'s own files relative to the package, and removes every other absolute path', () => { + const message = [ + 'Cannot find module \'/opt/checkly/dist/commands/deploy.js\'', + 'no config in /builds/acme/repo. Tried /builds/acme/repo/checkly.config.ts, C:\\ci\\x.ts and file:///srv/a.ts', + 'GET https://api.checklyhq.com/v1/checks failed', + ].join('\n') + + const scrubbed = scrubMessage(message, {}, posix) + + expect(scrubbed).toBe([ + 'Cannot find module \'checkly/dist/commands/deploy.js\'', + 'no config in Tried and ', + 'GET https://api.checklyhq.com/v1/checks failed', + ].join('\n')) + }) + + it('removes Windows paths in any case, with forward slashes, and as file URLs', () => { + const options = { + cwd: 'C:\\Users\\Ann Lee\\proj', + home: 'C:\\Users\\Ann Lee', + packageRoot: 'C:\\Program Files\\nodejs\\node_modules\\checkly', + platform: 'win32' as const, + } + const message = [ + 'c:/users/ann lee/proj/checkly.config.ts', + 'file:///C:/Users/Ann%20Lee/proj/a.check.ts', + 'C:\\Users\\Ann Lee\\other\\x.ts', + 'C:\\Program Files\\nodejs\\node_modules\\checkly\\dist\\index.js', + ].join('\n') + + const scrubbed = scrubMessage(message, {}, options) + + expect(scrubbed).toBe('\n\n\ncheckly/dist/index.js') + }) +}) + +describe('buildErrorEvent', () => { + it('names the CLI\'s files relative to the package and nothing about the machine', () => { + const event = buildErrorEvent(cliBug, { version: '7.1.0', command: 'deploy', cliMode: 'ci' }, { + packageRoot: installDir, + }) + + const [exception] = event.exception!.values! + expect(exception.type).toBe('TypeError') + const ownFrame = exception.stacktrace!.frames!.at(-1)! + expect(ownFrame).toMatchObject({ filename: 'checkly/dist/commands/deploy.mjs', function: 'run', in_app: true }) + expect(event).toMatchObject({ + release: 'checkly@7.1.0', + environment: 'production', + tags: { command: 'deploy', cli_mode: 'ci' }, + }) + + const serialized = JSON.stringify(event) + expect(serialized).not.toContain(installDir) + expect(serialized).not.toContain(os.homedir()) + expect(serialized).not.toContain(os.hostname()) + expect(event).not.toHaveProperty('server_name') + expect(event).not.toHaveProperty('user') + }) + + it('does not count a dependency under the CLI\'s own node_modules as CLI code', () => { + const event = buildErrorEvent(dependencyBug, { version: '7.1.0' }, { packageRoot: installDir }) + + const frame = event.exception!.values![0].stacktrace!.frames!.at(-1)! + expect(frame).toMatchObject({ filename: 'node_modules/some-dep/index.mjs', in_app: false }) + }) + + it('does not name a package the CLI does not depend on, which may be a private one of the user\'s', () => { + const event = buildErrorEvent(privatePackageBug, { version: '7.1.0' }, { packageRoot: installDir }) + + const frame = event.exception!.values![0].stacktrace!.frames!.at(-1)! + expect(frame).toEqual({ filename: '', function: '', in_app: false }) + expect(JSON.stringify(event)).not.toContain('acme-internal') + }) +}) + +describe('releaseEnvironment', () => { + it('tells releases apart by version', () => { + expect(releaseEnvironment('7.1.0')).toBe('production') + expect(releaseEnvironment('7.1.0-rc.2')).toBe('rc') + expect(releaseEnvironment('7.1.0-prerelease-abc1234')).toBe('prerelease') + }) +}) + +describe('isErrorReportingDisabled', () => { + it('honours CHECKLY_NO_ERROR_REPORTING and DO_NOT_TRACK, and not an explicit off', () => { + expect(isErrorReportingDisabled({})).toBe(false) + expect(isErrorReportingDisabled({ CHECKLY_NO_ERROR_REPORTING: '1' })).toBe(true) + expect(isErrorReportingDisabled({ DO_NOT_TRACK: '1' })).toBe(true) + expect(isErrorReportingDisabled({ DO_NOT_TRACK: '0' })).toBe(false) + }) +}) + +describe('reportNotedCliError', () => { + it('sends one envelope with the error to the DSN\'s project', async () => { + await withIngest(async (dsn, received) => { + await report(cliBug, { version: '7.1.0', command: 'deploy' }, { dsn, packageRoot: installDir, env: {} }) + // Once per run: a second error in the same run is not sent. + await report(cliBug, { version: '7.1.0', command: 'deploy' }, { dsn, packageRoot: installDir, env: {} }) + + expect(received).toHaveLength(1) + expect(received[0].url).toMatch(/^\/api\/123\/envelope\/\?.*sentry_key=a{72}/) + const [header, itemHeader, item] = received[0].body.split('\n').map(line => JSON.parse(line)) + expect(header.sdk).toMatchObject({ name: 'checkly-cli', version: '7.1.0' }) + expect(itemHeader.type).toBe('event') + expect(item.event_id).toBe(header.event_id) + expect(item.exception.values[0]).toMatchObject({ type: 'TypeError' }) + }) + }) + + it.each([ + ['the user turned reporting off', { version: '7.1.0' }, { DO_NOT_TRACK: '1' }], + ['the build is a development build', { version: '0.0.1-dev' }, {}], + ])('sends nothing when %s', async (_, context, env) => { + await withIngest(async (dsn, received) => { + await report(cliBug, context, { dsn, packageRoot: installDir, env }) + expect(received).toHaveLength(0) + + // The same call reports once the reason is gone, so the empty result above is not a broken server. + await report(cliBug, { version: '7.1.0' }, { dsn, packageRoot: installDir, env: {} }) + expect(received).toHaveLength(1) + }) + }) + + it('reports only the wrapped CLI bug, not the wrapper that quotes a user\'s file', async () => { + await withIngest(async (dsn, received) => { + const wrapper = new Error(`Error loading file '${installDir}/__checks__/api.check.ts'`, { cause: cliBug }) + await report(wrapper, { version: '7.1.0' }, { dsn, packageRoot: installDir, env: {} }) + + expect(received).toHaveLength(1) + expect(received[0].body).not.toContain('Error loading file') + const item = JSON.parse(received[0].body.split('\n')[2]) + expect(item.exception.values.map((value: { type: string }) => value.type)).toEqual(['TypeError']) + }) + }) + + it('sends the first error noted during the run when asked, as commands that print their errors need', async () => { + await withIngest(async (dsn, received) => { + noteCliError(new Error('Your project could not be deployed.')) + // A TypeError, but raised in a dependency: noted, then skipped when the run ends. + noteCliError(dependencyBug) + noteCliError(cliBug) + noteCliError(invalidUrl, { handled: false }) + expect(received).toHaveLength(0) + + await reportNotedCliError(() => ({ version: '7.1.0', command: 'deploy' }), { + dsn, + packageRoot: installDir, + env: {}, + }) + + expect(received).toHaveLength(1) + const item = JSON.parse(received[0].body.split('\n')[2]) + expect(item.exception.values[0].value).toBe(cliBug.message) + // The command printed it and exited, so it is not an unhandled crash. + expect(item.level).toBe('error') + expect(item.exception.values[0].mechanism).toEqual({ type: 'generic', handled: true }) + }) + }) + + it('reports an error that ended the command as an unhandled crash, and builds no context without one', async () => { + await withIngest(async (dsn, received) => { + let contextBuilt = false + const context = () => { + contextBuilt = true + return { version: '7.1.0' } + } + await reportNotedCliError(context, { dsn, packageRoot: installDir, env: {} }) + expect(contextBuilt).toBe(false) + + noteCliError(cliBug, { handled: false }) + await reportNotedCliError(context, { dsn, packageRoot: installDir, env: {} }) + + expect(contextBuilt).toBe(true) + expect(received).toHaveLength(1) + const item = JSON.parse(received[0].body.split('\n')[2]) + expect(item.level).toBe('fatal') + expect(item.exception.values[0].mechanism.handled).toBe(false) + }) + }) + + it('reads the DSN from the file the release writes, and sends nothing without it', async () => { + await withIngest(async (dsn, received) => { + const missing = path.join(installDir, 'missing', 'error-reporting.json') + await report(cliBug, { version: '7.1.0' }, { dsnFile: missing, packageRoot: installDir, env: {} }) + expect(received).toHaveLength(0) + + const dsnFile = path.join(installDir, 'error-reporting.json') + await fs.writeFile(dsnFile, JSON.stringify({ dsn })) + await report(cliBug, { version: '7.1.0' }, { dsnFile, packageRoot: installDir, env: {} }) + expect(received).toHaveLength(1) + }) + }) + + it('never throws when the ingest is unreachable', async () => { + await expect(report(cliBug, { version: '7.1.0' }, { + dsn: `http://${KEY}@127.0.0.1:1/123`, + packageRoot: installDir, + env: {}, + })).resolves.toBeUndefined() + }) +}) diff --git a/packages/cli/src/services/error-reporting/__tests__/sentry-protocol.spec.ts b/packages/cli/src/services/error-reporting/__tests__/sentry-protocol.spec.ts new file mode 100644 index 00000000..f2392ff0 --- /dev/null +++ b/packages/cli/src/services/error-reporting/__tests__/sentry-protocol.spec.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest' + +import { envelopeEndpoint, eventId, parseStack, serializeEventEnvelope, type ErrorEvent } from '../sentry-protocol.js' + +// The expected values are what @sentry/core 11 produced for the same input, +// except where a test says otherwise: only lines starting with `at` are +// frames, and only file URLs are decoded. +describe('parseStack', () => { + it('returns the frames outermost first', () => { + const frames = parseStack([ + 'TypeError: boom', + ' at inner (/abs/inner.js:1:2)', + ' at outer (/abs/outer.js:3:4)', + ].join('\n')) + expect(frames).toEqual([ + { filename: '/abs/outer.js', function: 'outer', lineno: 3, colno: 4 }, + { filename: '/abs/inner.js', function: 'inner', lineno: 1, colno: 2 }, + ]) + }) + + it('turns file URLs into paths, with spaces and Windows drives', () => { + const [windows, posix] = parseStack([ + ' at Deploy.run (file:///Users/ann/my%20cli/dist/commands/deploy.js:12:34)', + ' at async Deploy._run (file:///C:/Users/Ann%20Lee/cli/dist/x.js:1:1)', + ].join('\n')) + expect(posix).toMatchObject({ filename: '/Users/ann/my cli/dist/commands/deploy.js', function: 'Deploy.run' }) + expect(windows).toMatchObject({ filename: 'C:/Users/Ann Lee/cli/dist/x.js', function: 'Deploy._run' }) + }) + + it('keeps native Windows paths as they are', () => { + expect(parseStack(' at run (C:\\Users\\Ann Lee\\cli\\dist\\y.js:5:6)')).toEqual([ + { filename: 'C:\\Users\\Ann Lee\\cli\\dist\\y.js', function: 'run', lineno: 5, colno: 6 }, + ]) + }) + + it('names constructors, anonymous functions and frames without a function', () => { + const frames = parseStack([ + ' at new Foo (/abs/foo.js:7:8)', + ' at Object. (/abs/cjs.js:1:1)', + ' at /abs/anon.js:2:3', + ].join('\n')).reverse() + expect(frames.map(frame => frame.function)).toEqual(['new Foo', 'Object.?', '?']) + expect(frames[2].filename).toBe('/abs/anon.js') + }) + + it('keeps the eval origin as the filename of an eval frame', () => { + expect(parseStack(' at eval (eval at load (/abs/loader.js:1:2), :1:1)')).toEqual([ + { filename: 'eval at load (/abs/loader.js:1:2), ', function: 'eval', lineno: 1, colno: 1 }, + ]) + }) + + it('keeps frames without a line, and Node\'s own frames', () => { + const frames = parseStack([ + ' at Array.map ()', + ' at async Promise.all (index 0)', + ' at process.processTicksAndRejections (node:internal/process/task_queues:95:5)', + ].join('\n')).reverse() + expect(frames).toEqual([ + { filename: '', function: 'Array.map' }, + { filename: 'index 0', function: 'Promise.all' }, + { + filename: 'node:internal/process/task_queues', + function: 'process.processTicksAndRejections', + lineno: 95, + colno: 5, + }, + ]) + }) + + it('skips lines that are not frames, even a message line that mentions one', () => { + const frames = parseStack([ + 'TypeError: Cannot read properties of undefined (reading \'x\')', + 'second message line at foo (/tmp/msg.js:1:2)', + '', + ' at run (/abs/run.js:1:1)', + ].join('\n')) + expect(frames.map(frame => frame.filename)).toEqual(['/abs/run.js']) + }) + + it('skips lines too long to parse rather than cutting them', () => { + const frames = parseStack([ + ` at fn (/abs/${'a'.repeat(1100)}/node_modules/dep/x.js:1:2)`, + ' at run (/abs/run.js:1:1)', + ].join('\n')) + expect(frames.map(frame => frame.filename)).toEqual(['/abs/run.js']) + }) +}) + +describe('eventId', () => { + it('is 32 hex characters', () => { + expect(eventId()).toMatch(/^[0-9a-f]{32}$/) + }) +}) + +describe('envelopeEndpoint', () => { + it('builds the ingest URL with the key in the query string', () => { + expect(envelopeEndpoint('https://aaaaaaaa@ingest.example.com:8443/sub/path/123')) + .toBe('https://ingest.example.com:8443/sub/path/api/123/envelope/?sentry_version=7&sentry_key=aaaaaaaa') + expect(envelopeEndpoint('http://key@127.0.0.1:9/123')) + .toBe('http://127.0.0.1:9/api/123/envelope/?sentry_version=7&sentry_key=key') + }) + + it.each(['ftp://k@h/1', 'https://h/1', 'https://k@h/abc', 'https://k@h/1/', 'not a dsn'])( + 'rejects %s', dsn => { + expect(envelopeEndpoint(dsn)).toBeUndefined() + }) +}) + +describe('serializeEventEnvelope', () => { + it('writes the envelope headers, the item header and the event, one per line', () => { + const event = { event_id: 'e'.repeat(32), level: 'error' } as ErrorEvent + const lines = serializeEventEnvelope(event, { name: 'checkly-cli', version: '7.1.0' }).split('\n') + + expect(lines).toHaveLength(3) + const [headers, itemHeaders, payload] = lines.map(line => JSON.parse(line)) + expect(headers).toEqual({ + event_id: 'e'.repeat(32), + sent_at: expect.stringMatching(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/), + sdk: { name: 'checkly-cli', version: '7.1.0' }, + }) + expect(itemHeaders).toEqual({ type: 'event' }) + expect(payload).toEqual({ + event_id: 'e'.repeat(32), + level: 'error', + sdk: { name: 'checkly-cli', version: '7.1.0', integrations: [], packages: [] }, + }) + }) +}) diff --git a/packages/cli/src/services/error-reporting/classify.ts b/packages/cli/src/services/error-reporting/classify.ts new file mode 100644 index 00000000..d76923be --- /dev/null +++ b/packages/cli/src/services/error-reporting/classify.ts @@ -0,0 +1,128 @@ +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { parseStack, type StackFrame } from './sentry-protocol.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +// This file compiles to dist/services/error-reporting/classify.js. +export const PACKAGE_ROOT = path.resolve(__dirname, '..', '..', '..') + +// The runtime errors a bug in the CLI's own code throws. Every other error is +// either one the CLI throws on purpose to tell the user something (invalid +// input, a failed deploy, an API error) or one from the user's system (files, +// network), and none of those are bugs in the CLI. +const REPORTED_ERROR_NAMES = new Set(['TypeError', 'ReferenceError', 'RangeError']) + +// TypeErrors Node raises for input the user gave the CLI, e.g. a malformed +// CHECKLY_API_URL or proxy URL: the CLI's code is fine, the value is not. +const USER_INPUT_ERROR_CODES = new Set(['ERR_INVALID_URL', 'ERR_INVALID_URL_SCHEME']) + +// How deep to look into `cause` for the error a wrapper hides, e.g. the +// TypeError in `Error loading file '…'`. +export const MAX_CAUSE_DEPTH = 5 + +export function realpath (p: string): string { + try { + return fs.realpathSync(p) + } catch { + return p + } +} + +function isInside (file: string, dir: string): boolean { + const relative = path.relative(dir, file) + return !!relative && relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative) +} + +/** Whether `file` is one of the CLI's own files, not one of its dependencies. */ +export function isOwnPath (file: string, packageRoot: string): boolean { + return isInside(file, packageRoot) && !isInside(file, path.join(packageRoot, 'node_modules')) +} + +/** Whether `err` is of a type a CLI bug throws. Looks only at the error, not at its stack. */ +function isBugType (err: unknown): err is Error { + return err instanceof Error + && REPORTED_ERROR_NAMES.has(err.name) + && !USER_INPUT_ERROR_CODES.has((err as NodeJS.ErrnoException).code ?? '') +} + +function* errorChain (err: unknown): Generator { + let current = err + for (let depth = 0; current instanceof Error && depth < MAX_CAUSE_DEPTH; depth++) { + yield current + current = current.cause + } +} + +/** + * Whether `err` or one of its causes is of a type a CLI bug throws. Cheap, so + * it can run where an error is printed; `findCliBug`, which reads the stack, + * decides. + */ +export function mayBeCliBug (err: unknown): boolean { + for (const error of errorChain(err)) { + if (isBugType(error)) return true + } + return false +} + +// Code run through eval or `new Function` names its frame +// `eval at load (/path/file.js:1:2), `, which parseStack keeps as +// the filename. The file in the innermost parentheses is the one that ran it. +const EVAL_ORIGIN = /\(([^()]+):\d+:\d+\)/ + +/** The file a frame's code is in, as a path. */ +function frameFile (filename: string): string { + const file = filename.startsWith('eval at ') ? EVAL_ORIGIN.exec(filename)?.[1] ?? filename : filename + try { + return file.startsWith('file:') ? fileURLToPath(file) : file + } catch { + return file + } +} + +/** Whether `file` names a file: not ``, `index 0` or a `node:` module. */ +function isFile (file: string): boolean { + return path.isAbsolute(file) || path.win32.isAbsolute(file) +} + +/** + * The file of the frame that threw: the innermost one in a file. A built-in + * that throws, e.g. `'x'.repeat(-1)`, adds a frame like `String.repeat + * ()` and Node's own modules add `node:` frames; the code that + * called them decides. + */ +function throwingFile (frames: StackFrame[]): string | undefined { + // parseStack orders frames outermost first. + for (let i = frames.length - 1; i >= 0; i--) { + const filename = frames[i].filename + const file = filename && frameFile(filename) + if (file && isFile(file)) return file + } + return undefined +} + +/** + * Whether `err` is a bug in the CLI's own code: one of the runtime error types + * a bug throws, raised in a file of the CLI. A TypeError in a user's check + * file, which the CLI loads and runs in its own process, is the user's. + */ +export function isCliBug (err: unknown, packageRoot: string = PACKAGE_ROOT): boolean { + if (!isBugType(err)) return false + const file = throwingFile(parseStack(err.stack ?? '')) + return !!file && isOwnPath(file, realpath(packageRoot)) +} + +/** + * The CLI bug in `err` or in its causes: the CLI wraps many errors in a plain + * Error with the original as `cause`. Only that error is reported, not the + * wrapper, whose message may quote a user's file and its stack. + */ +export function findCliBug (err: unknown, packageRoot: string = PACKAGE_ROOT): Error | undefined { + for (const error of errorChain(err)) { + if (isCliBug(error, packageRoot)) return error + } + return undefined +} diff --git a/packages/cli/src/services/error-reporting/event.ts b/packages/cli/src/services/error-reporting/event.ts new file mode 100644 index 00000000..e8bbf834 --- /dev/null +++ b/packages/cli/src/services/error-reporting/event.ts @@ -0,0 +1,78 @@ +import { MAX_CAUSE_DEPTH, PACKAGE_ROOT, realpath } from './classify.js' +import { cliDependencies, scrubFrame, scrubMessage } from './scrub.js' +import { eventId, parseStack, type ErrorEvent, type Exception } from './sentry-protocol.js' + +export interface ErrorReportContext { + /** The CLI version, `this.config.version`. */ + version: string + /** The oclif command id, e.g. `deploy` or `checks:list`. */ + command?: string + /** The detected CLI mode: interactive, ci or agent. */ + cliMode?: string + /** Whether a command caught and printed the error, rather than it ending the command. */ + handled?: boolean +} + +/** The environment a release reports as, from its version. */ +export function releaseEnvironment (version: string): string { + if (/^\d+\.\d+\.\d+$/.test(version)) return 'production' + if (/-rc\.\d+$/.test(version)) return 'rc' + // Prereleases (`-prerelease-`) and anything else. Canary and local + // builds never report (isDevelopmentBuild). + return 'prerelease' +} + +/** + * The exceptions of `err` and its causes, innermost cause first, as Sentry + * expects them. Only the name, the scrubbed message and the scrubbed frames + * are kept: an error's other properties (an axios error's request config with + * its auth headers, for one) are never sent. + */ +function toExceptions (err: Error, packageRoot: string, env: NodeJS.ProcessEnv, handled: boolean): Exception[] { + const dependencies = cliDependencies(packageRoot) + + const exceptions: Exception[] = [] + let current: unknown = err + while (current instanceof Error && exceptions.length < MAX_CAUSE_DEPTH) { + const frames = parseStack(current.stack ?? '') + exceptions.unshift({ + type: current.name, + value: scrubMessage(current.message, env, { packageRoot }), + stacktrace: frames.length + ? { frames: frames.map(frame => scrubFrame(frame, packageRoot, dependencies)) } + : undefined, + mechanism: { type: 'generic', handled }, + }) + current = current.cause + } + return exceptions +} + +/** The Sentry event for `err`. Exported for tests. */ +export function buildErrorEvent ( + err: Error, + context: ErrorReportContext, + { packageRoot = PACKAGE_ROOT, env = process.env }: { packageRoot?: string, env?: NodeJS.ProcessEnv } = {}, +): ErrorEvent { + const runtime = process.versions.bun ? 'bun' : process.versions.deno ? 'deno' : 'node' + return { + event_id: eventId(), + timestamp: Date.now() / 1000, + platform: 'node', + level: context.handled ? 'error' : 'fatal', + release: `checkly@${context.version}`, + environment: releaseEnvironment(context.version), + exception: { values: toExceptions(err, realpath(packageRoot), env, !!context.handled) }, + tags: { + ...(context.command && { command: context.command }), + ...(context.cliMode && { cli_mode: context.cliMode }), + }, + contexts: { + os: { name: process.platform }, + runtime: { name: runtime, version: process.versions[runtime] ?? process.version }, + device: { arch: process.arch }, + }, + // Only what is set above is sent: no computer name, user, locale, working + // directory, command arguments or environment variables. + } +} diff --git a/packages/cli/src/services/error-reporting/report.ts b/packages/cli/src/services/error-reporting/report.ts new file mode 100644 index 00000000..adb87758 --- /dev/null +++ b/packages/cli/src/services/error-reporting/report.ts @@ -0,0 +1,137 @@ +import fs from 'node:fs' +import path from 'node:path' +import axios from 'axios' +import Debug from 'debug' + +import { assignProxy } from '../proxy.js' +import { isEnvFlagSet } from '../../helpers/cli-mode.js' +import { isDevelopmentBuild } from '../project-cli.js' +import { findCliBug, mayBeCliBug, PACKAGE_ROOT } from './classify.js' +import { buildErrorEvent, type ErrorReportContext } from './event.js' +import { envelopeEndpoint, serializeEventEnvelope } from './sentry-protocol.js' + +const debug = Debug('checkly:error-reporting') + +// Written to the package root by the release workflow before `pnpm pack`, which +// rebuilds dist. Builds without it (local checkouts, forks, tests) report nothing. +const DSN_FILE = path.join(PACKAGE_ROOT, 'error-reporting.json') + +// One small POST; the error is printed after it, so the user waits this long at most. +const SEND_TIMEOUT_MS = 1500 + +// Enough to get past the ordinary errors a command prints before its bug. +const MAX_NOTED_ERRORS = 5 + +export interface ErrorReportOptions { + /** The DSN to report to. Defaults to the one in `dsnFile`. */ + dsn?: string + /** The file the release workflow writes the DSN to. */ + dsnFile?: string + /** The CLI's install directory; frames inside it are the CLI's own code. */ + packageRoot?: string + env?: NodeJS.ProcessEnv +} + +interface Target { + dsn: string + packageRoot: string + env: NodeJS.ProcessEnv +} + +let reported = false +let noted: Array<{ error: unknown, handled: boolean }> = [] +let defaultOptions: ErrorReportOptions = {} + +/** Whether the user turned error reporting off with `CHECKLY_NO_ERROR_REPORTING` or `DO_NOT_TRACK`. */ +export function isErrorReportingDisabled (env: NodeJS.ProcessEnv = process.env): boolean { + return isEnvFlagSet(env.CHECKLY_NO_ERROR_REPORTING) || isEnvFlagSet(env.DO_NOT_TRACK) +} + +function readReleaseDsn (file: string): string | undefined { + try { + const { dsn } = JSON.parse(fs.readFileSync(file, 'utf8')) + return typeof dsn === 'string' && dsn ? dsn : undefined + } catch { + return undefined + } +} + +/** Where to send a report, or `undefined` when this run must not send one. */ +function reportTarget (version: string, options: ErrorReportOptions): Target | undefined { + const env = options.env ?? process.env + if (reported || isDevelopmentBuild(version) || isErrorReportingDisabled(env)) return undefined + const dsn = options.dsn ?? readReleaseDsn(options.dsnFile ?? DSN_FILE) + return dsn ? { dsn, env, packageRoot: options.packageRoot ?? PACKAGE_ROOT } : undefined +} + +async function send (bug: Error, context: ErrorReportContext, { dsn, packageRoot, env }: Target): Promise { + reported = true + const url = envelopeEndpoint(dsn) + if (!url) return + const event = buildErrorEvent(bug, context, { packageRoot, env }) + const envelope = serializeEventEnvelope(event, { name: 'checkly-cli', version: context.version }) + const { status } = await axios.post(url, envelope, assignProxy(url, { + headers: { 'Content-Type': 'application/x-sentry-envelope' }, + timeout: SEND_TIMEOUT_MS, + // `timeout` only limits the wait for a response; this also ends a + // connection that never opens, e.g. behind a proxy that drops it. + signal: AbortSignal.timeout(SEND_TIMEOUT_MS), + validateStatus: () => true, + })) + if (status >= 200 && status < 300) { + debug('Reported %s as %s', bug.name, event.event_id) + } else { + debug('The ingest rejected the report of %s with status %d', bug.name, status) + } +} + +/** + * Remembers an error for `reportNotedCliError` to check and send when the + * command ends. For errors a command catches and prints itself instead of + * throwing, e.g. `deploy` and `test`, which never reach `BaseCommand.catch`. + */ +export function noteCliError (err: unknown, { handled = true }: { handled?: boolean } = {}): void { + try { + if (noted.length < MAX_NOTED_ERRORS && mayBeCliBug(err)) noted.push({ error: err, handled }) + } catch { + // Never let reporting change how a command fails. + } +} + +/** + * Sends the first noted error that is a CLI bug, if any. Reports at most one + * error per run, waits at most a few seconds, and never throws: a failed + * report must not change how the command fails. + */ +export async function reportNotedCliError ( + context: () => Omit, + options: ErrorReportOptions = defaultOptions, +): Promise { + try { + if (noted.length === 0) return + const base = context() + const target = reportTarget(base.version, options) + if (!target) return + for (const { error, handled } of noted) { + const bug = findCliBug(error, target.packageRoot) + if (bug) return await send(bug, { ...base, handled }, target) + } + } catch (reportError) { + debug('Could not report the error: %O', reportError) + } +} + +/** + * Sets the options `reportNotedCliError` uses when it is given none, as in + * `BaseCommand.finally`. For tests, which have no release DSN file. + */ +export function setDefaultErrorReportOptions (options: ErrorReportOptions): void { + defaultOptions = options +} + +/** Lets the next report go out again. For tests. */ +export function resetErrorReporting (): void { + reported = false + noted = [] + defaultOptions = {} +} diff --git a/packages/cli/src/services/error-reporting/scrub.ts b/packages/cli/src/services/error-reporting/scrub.ts new file mode 100644 index 00000000..8b51ab81 --- /dev/null +++ b/packages/cli/src/services/error-reporting/scrub.ts @@ -0,0 +1,169 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { isOwnPath, PACKAGE_ROOT } from './classify.js' +import type { StackFrame } from './sentry-protocol.js' + +/** The packages the CLI depends on directly; frames in other packages are the user's. */ +export function cliDependencies (packageRoot: string): Set { + try { + const { dependencies = {} } = JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + return new Set(Object.keys(dependencies)) + } catch { + return new Set() + } +} + +/** `axios` for `node_modules/axios/lib/x.js`, `@scope/name` for a scoped package. */ +function packageName (afterNodeModules: string): string { + const [first, second] = afterNodeModules.split('/') + return first.startsWith('@') ? `${first}/${second}` : first +} + +/** + * Rewrites a frame's path so it says where in the CLI or its dependencies the + * code is, and nothing about the machine: no home directory, no user name, no + * project location. A file of the user's own (a check, the config) is not + * named at all. + */ +export function scrubFrame (frame: StackFrame, packageRoot: string, dependencies: Set): StackFrame { + const filename = frame.filename + if (!filename || filename.startsWith('node:')) { + return { ...frame, in_app: false } + } + if (isOwnPath(filename, packageRoot)) { + const relative = path.relative(packageRoot, filename).split(path.sep).join('/') + return { ...frame, filename: `checkly/${relative}`, in_app: true } + } + const normalized = filename.split(path.sep).join('/') + const nodeModules = normalized.lastIndexOf('node_modules/') + if (nodeModules !== -1) { + const inPackage = normalized.slice(nodeModules + 'node_modules/'.length) + if (!dependencies.has(packageName(inPackage))) { + // A package of the user's, possibly a private one: its name is not sent. + return { filename: '', function: '', in_app: false } + } + return { ...frame, filename: normalized.slice(nodeModules), in_app: false } + } + return { filename: '', function: '', in_app: false } +} + +export interface ScrubOptions { + cwd?: string + home?: string + packageRoot?: string + platform?: NodeJS.Platform +} + +function escapeRegExp (text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +/** The ways a message may spell `dir`: native, with forward slashes, and as a file URL. */ +function pathSpellings (dir: string): string[] { + const forward = dir.replace(/\\/g, '/') + const url = `file://${forward.startsWith('/') ? '' : '/'}${encodeURI(forward)}` + return [...new Set([dir, forward, url])] +} + +// What may follow a path in a message: a separator, a closing bracket or +// quote, or a full stop that ends a sentence. +const PATH_END = '(?=[\\s\'"`():<>,;!\\]}]|\\.(?:\\s|$)|$)' + +/** + * Replaces `dir`, and any path inside it, with `replacement`, which gets the + * part after `dir`. Only whole path segments match: a working directory `/app` + * leaves `/application/json` alone. + */ +function replaceDir ( + message: string, + dir: string, + replacement: (rest: string) => string, + platform: NodeJS.Platform, +): string { + if (!dir || dir.length < 2) return message + const spellings = pathSpellings(dir.replace(/[\\/]+$/, '')).map(escapeRegExp).join('|') + const pattern = new RegExp(`(?:${spellings})([\\\\/][^\\s'"\`()<>]*)?${PATH_END}`, + platform === 'win32' ? 'gi' : 'g') + return message.replace(pattern, (_, rest: string | undefined) => replacement(rest ?? '')) +} + +// Characters that end a path in a message. \x60 is a backtick. +const NOT_PATH = String.raw`\s'"\x60()<>` + +// Any other absolute path, e.g. a CI checkout outside the working directory +// or a --config file elsewhere, at the start of a word. `https://host/path` is +// a URL and stays. +const ABSOLUTE_PATH = new RegExp(String.raw`(^|[\s'"\x60(=,[{<])(?:` + [ + String.raw`file:\/\/[^${NOT_PATH}]+`, + String.raw`\/(?=[^/${NOT_PATH}])[^${NOT_PATH}]*`, + String.raw`[A-Za-z]:[\\/][^${NOT_PATH}]*`, + // A UNC path, \\server\share. + String.raw`\\\\[^${NOT_PATH}]+`, +].join('|') + ')', 'g') + +// A value given to a name that says it is secret: `token=abc`, `"password": "a b"`, +// also access_token, client_secret, accessToken, x-api-key and the like. +const SECRET_NAME = String.raw`[\w-]*(?:api[_-]?key|token|secret|password|passwd)` +const SECRET_VALUE = new RegExp(String.raw`(${SECRET_NAME}["']?\s*[:=]\s*)("[^"]*"|'[^']*'|["']?[^\s"'&,;]+)`, 'gi') + +// A URL with an `@` somewhere, which may carry a user and password. Before +// the last `@`, any character may be part of a password; after it, a quote or +// bracket ends the URL. +const URL_WITH_AT = /\b[a-z][a-z0-9+.-]*:\/\/\S*@[^\s'"\x60<>]*/gi + +/** + * A URL with credentials rebuilt from its scheme and host alone, and one that + * cannot be parsed withheld: a password may hold any character, so no pattern + * finds where it ends. + */ +function withoutUrlCredentials (url: string): string { + let parsed: URL + try { + parsed = new URL(url) + } catch { + return `${url.slice(0, url.indexOf('://'))}://` + } + if (!parsed.username && !parsed.password) return url + return `${parsed.protocol}//@${parsed.host}` +} + +/** ``, inside the quotes the value had. */ +function redactValue (value: string): string { + const quote = value[0] + if (quote !== '"' && quote !== '\'') return '' + return value.length > 1 && value.endsWith(quote) ? `${quote}${quote}` : `${quote}` +} + +/** + * Removes what an error message may carry about the user: credentials, and + * paths that name the user, their project or its files. + */ +export function scrubMessage ( + message: string, + env: NodeJS.ProcessEnv = process.env, + { + cwd = process.cwd(), + home = os.homedir(), + packageRoot = PACKAGE_ROOT, + platform = process.platform, + }: ScrubOptions = {}, +): string { + let scrubbed = message + // Checkly API keys. + .replace(/\bcu_[A-Za-z0-9]{8,}\b/g, '') + .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, '$1 ') + .replace(URL_WITH_AT, withoutUrlCredentials) + .replace(SECRET_VALUE, (_, name: string, value: string) => name + redactValue(value)) + for (const value of [env.CHECKLY_API_KEY, env.CHECKLY_ACCOUNT_ID]) { + if (value && value.length >= 8) scrubbed = scrubbed.split(value).join('') + } + // The CLI's own files first, as frames name them: the CLI may be installed + // in the project or the home directory. + scrubbed = replaceDir(scrubbed, packageRoot, rest => `checkly${rest.replace(/\\/g, '/')}`, platform) + // The project next: it is usually inside the home directory. + scrubbed = replaceDir(scrubbed, cwd, () => '', platform) + scrubbed = replaceDir(scrubbed, home, () => '', platform) + return scrubbed.replace(ABSOLUTE_PATH, '$1') +} diff --git a/packages/cli/src/services/error-reporting/sentry-protocol.ts b/packages/cli/src/services/error-reporting/sentry-protocol.ts new file mode 100644 index 00000000..429cf93c --- /dev/null +++ b/packages/cli/src/services/error-reporting/sentry-protocol.ts @@ -0,0 +1,124 @@ +import { randomUUID } from 'node:crypto' + +// The parts of the Sentry protocol the CLI's crash reports need: Checkly's +// ingest accepts Sentry envelopes. https://develop.sentry.dev/sdk/data-model/ + +export interface StackFrame { + filename?: string + function?: string + lineno?: number + colno?: number + in_app?: boolean +} + +export interface Exception { + type: string + value: string + stacktrace?: { frames: StackFrame[] } + mechanism: { type: string, handled: boolean } +} + +export interface ErrorEvent { + event_id: string + timestamp: number + platform: string + level: 'error' | 'fatal' + release: string + environment: string + exception: { values: Exception[] } + tags: Record + contexts: Record> +} + +export interface SdkInfo { + name: string + version: string +} + +// More frames than Node records by default (10), and enough for a report. +const MAX_FRAMES = 50 + +// Longer lines are skipped, to bound the regular expression's work. Cutting +// them instead would turn a long path into a shorter, wrong one. +const MAX_LINE_LENGTH = 1024 + +// A V8 stack line: `at [async] [function (]location[)]`, where the location is +// `file:line:column`, or e.g. ``, `native` or `index 0` without one. +const FRAME_LINE = /^\s*at (?:async )?(?:(.+?)\s+\()?(?:(.+):(\d+):(\d+)?|([^)]+))\)?/ + +/** A frame's file as a path: `file:///C:/Ann%20Lee/x.js` is `C:/Ann Lee/x.js`. Other names stay as they are. */ +function framePath (location: string): string { + if (!location.startsWith('file://')) return location + let file = location.slice('file://'.length) + if (/^\/[A-Za-z]:/.test(file)) file = file.slice(1) + try { + return decodeURI(file) + } catch { + return file + } +} + +/** `?` for a function V8 has no name for, as in `Object.`. */ +function functionName (name: string | undefined): string { + if (!name) return '?' + return name.endsWith('.') ? `${name.slice(0, -''.length)}?` : name +} + +function parseFrame (line: string): StackFrame | undefined { + if (line.length > MAX_LINE_LENGTH) return undefined + const match = FRAME_LINE.exec(line) + if (!match) return undefined + const [, name, file, lineno, colno, location] = match + const where = file ?? (location === 'native' ? undefined : location) + return { + filename: where && framePath(where), + function: functionName(name), + lineno: Number(lineno) || undefined, + colno: Number(colno) || undefined, + } +} + +/** + * The frames of a V8 `err.stack`, outermost first as Sentry expects them. + * Lines that are not frames, such as the message, are skipped. + */ +export function parseStack (stack: string): StackFrame[] { + const frames: StackFrame[] = [] + for (const line of stack.split('\n')) { + const frame = parseFrame(line) + if (frame) frames.push(frame) + if (frames.length >= MAX_FRAMES) break + } + return frames.reverse() +} + +/** A Sentry event id: 32 hex characters. */ +export function eventId (): string { + return randomUUID().replace(/-/g, '') +} + +/** + * The URL to post envelopes to for a DSN `https://@[/]/`, + * with the key in the query string, or `undefined` for a DSN that is not one. + */ +export function envelopeEndpoint (dsn: string): string | undefined { + let url: URL + try { + url = new URL(dsn) + } catch { + return undefined + } + const segments = url.pathname.slice(1).split('/') + const projectId = /^\d+/.exec(segments.pop() ?? '')?.[0] + if (!['http:', 'https:'].includes(url.protocol) || !url.username || !projectId) return undefined + const path = segments.length ? `/${segments.join('/')}` : '' + const query = new URLSearchParams({ sentry_version: '7', sentry_key: url.username }) + return `${url.protocol}//${url.host}${path}/api/${projectId}/envelope/?${query}` +} + +/** An envelope holding just `event`: its headers, the item's header and the event, one JSON document per line. */ +export function serializeEventEnvelope (event: ErrorEvent, sdk: SdkInfo): string { + const headers = { event_id: event.event_id, sent_at: new Date().toISOString(), sdk } + const payload = { ...event, sdk: { ...sdk, integrations: [], packages: [] } } + return [headers, { type: 'event' }, payload].map(part => JSON.stringify(part)).join('\n') +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4b7a00a0..d6dbaba9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1178,6 +1178,7 @@ packages: '@smithy/core@3.35.2': resolution: {integrity: sha512-ejTukBuP/Pg3t9N8Ux+0zKsXeHWm7p6W3HbXFOyltKasSxZwCFmvTut7kBdZSCF3m9y1bXXyVu6p2xuMFcsTyQ==} engines: {node: '>=18.0.0'} + deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==}