diff --git a/packages/core/src/helpers/request.ts b/packages/core/src/helpers/request.ts index 9ea510976..396043234 100644 --- a/packages/core/src/helpers/request.ts +++ b/packages/core/src/helpers/request.ts @@ -96,7 +96,7 @@ export const NB_RETRIES = 5; // Do a retriable fetch. export const doRequest = async (opts: RequestOpts): Promise => { - const { auth, url, method = 'GET', getData, type = 'text' } = opts; + const { auth, url, method = 'GET', getData, type = 'text', onResponse } = opts; const retryOpts: retry.Options = { retries: opts.retries === 0 ? 0 : opts.retries || NB_RETRIES, onRetry: opts.onRetry, @@ -168,6 +168,9 @@ export const doRequest = async (opts: RequestOpts): Promise => { } } + // Expose the successful response (headers, status) before the body is consumed. + onResponse?.(response); + try { let result; // Await it so we catch any parsing error and bail. diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 63e8fab9f..02dc2cfdb 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -310,6 +310,9 @@ export type RequestOpts = { getData?: () => Promise | Data; type?: 'json' | 'text'; onRetry?: (error: Error, attempt: number) => void; + // Invoked with the raw response of each successful (HTTP-ok) attempt, before the + // body is consumed. Useful to read response headers (e.g. ETag, Last-Modified). + onResponse?: (response: Response) => void; retries?: number; minTimeout?: number; maxTimeout?: number; diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index 231439d49..5ea024d86 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -42,7 +42,7 @@ When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./ ## Runtime benchmark -The opt-in browser benchmark measures the dormant runtime overhead added by Live Debugger instrumentation. It compares instrumented code against equivalent uninstrumented code, back-to-back in the same browser session, while SDK-like dormant probe hooks are installed. +The opt-in browser benchmark measures the dormant runtime overhead added by Live Debugger instrumentation. It compares instrumented code against equivalent uninstrumented code, back-to-back in the same browser session, with the real Browser Debugger SDK loaded but dormant (no active probes). ### Running it @@ -66,7 +66,7 @@ Each sample measures three variants: - **baseline**: the uninstrumented workload. - **control**: the same baseline function measured a second time. This is an A/A diagnostic for timing noise in the benchmark apparatus. -- **instrumented**: the same workload after Live Debugger instrumentation, with dormant SDK hooks installed. +- **instrumented**: the same workload after Live Debugger instrumentation, with the real Browser Debugger SDK loaded but dormant (no active probes). The reporter estimates overhead from `instrumented - control`. That direct paired difference avoids the old correlated-interval comparison against the shared baseline sample. The `control - baseline` result is still shown as the A/A diagnostic; it should be centered around zero if the browser session is quiet enough to trust. @@ -110,6 +110,8 @@ The benchmark tries to make each browser comparison fair and repeatable: The reported point estimate is a trimmed mean of `instrumented - control`: the benchmark drops the noisiest 20% on each side and averages the middle. Confidence intervals are bootstrapped from the same paired samples. The percentage column uses the same paired data, but divides by the baseline workload time. +The benchmark uses the real published Browser Debugger SDK, loaded in a dormant state with no active probes, so the measured runtime path matches what ships. + The code uses more specific statistical machinery than this section describes, but the practical rule is simple: trust `clean` rows, rerun `unreliable` rows, and compare `Tiny` and `Hot` primarily on `per-call overhead upper`. ### Caveats diff --git a/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts index 5de42dc84..52b3c4eea 100644 --- a/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts +++ b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts @@ -5,37 +5,37 @@ /* eslint-env browser */ /* global globalThis */ import { existsSync, mkdir, outputFile, rm } from '@dd/core/helpers/fs'; +import { doRequest } from '@dd/core/helpers/request'; import { verifyProjectBuild } from '@dd/tests/_playwright/helpers/buildProject'; import type { TestOptions } from '@dd/tests/_playwright/testParams'; import { test } from '@dd/tests/_playwright/testParams'; import { defaultConfig } from '@dd/tools/plugins'; import type { Page } from '@playwright/test'; import fs from 'fs'; +import nock from 'nock'; import path from 'path'; import { getLiveDebuggerBenchConfig } from './liveDebuggerBenchConfig'; -import type { BenchVariant, BrowserBenchApi } from './types'; +import { normalizeEtag } from './reporter/benchReporter'; +import type { BenchVariant, BrowserBenchApi, SdkBuild } from './types'; /* eslint-disable no-var, vars-on-top */ declare global { var ddBench: Record | undefined; - var ddActiveProbes: Record; var crossOriginIsolated: boolean; var $dd_probes: (functionId: string) => unknown[] | undefined; - var $dd_entry: (probes: unknown[], self: unknown, args?: Record) => void; - var $dd_return: ( - probes: unknown[], - value: unknown, - self: unknown, - args?: Record, - locals?: Record, - ) => unknown; - var $dd_throw: ( - probes: unknown[], - error: unknown, - self: unknown, - args?: Record, - ) => void; + var DD_DEBUGGER: + | { + version: string; + init: (configuration: { + clientToken: string; + service: string; + site?: string; + proxy?: string; + pollInterval?: number; + }) => void; + } + | undefined; } /* eslint-enable no-var, vars-on-top */ @@ -47,6 +47,16 @@ const BENCH_OPTIONS = { warmupMs: process.env.CI ? 250 : 300, }; +// The Browser Debugger SDK is published to the CDN on every commit to master, so the +// benchmark exercises the real, shipped $dd_probes / $dd_entry / $dd_return / $dd_throw +// instead of a hand-written stub that could silently drift from the SDK. +const SDK_BUNDLE_URL = 'https://www.datadoghq-browser-agent.com/us1/v7/datadog-debugger.js'; +// Cache the vendored SDK bundle under the build's dist/ folder so it inherits the repo's +// ignore rules for build output (it is minified and must not be linted or committed). +const SDK_BUNDLE_RELATIVE_PATH = path.join('dist', 'datadog-debugger.cdn.js'); +// Probe-delivery endpoint the SDK polls, hardcoded in the SDK's delivery API. +const DEBUGGER_PROBES_PATH = '/api/unstable/debugger/frontend/probes'; + const copyFixtureShell = async (source: string, destination: string) => { await fs.promises.cp(`${source}/`, `${destination}/`, { recursive: true, @@ -132,24 +142,113 @@ const userFlow = async (url: string, page: Page, bundler: TestOptions['bundler'] }); }; -const installDormantDebuggerSdkHooks = async (page: Page) => { - await page.addInitScript(() => { - globalThis.ddActiveProbes = { - // Mirror the Browser SDK by adding a __placeholder__ key. - __placeholder__: undefined, - }; - globalThis.$dd_probes = (functionId) => globalThis.ddActiveProbes[functionId]; - globalThis.$dd_entry = () => {}; - globalThis.$dd_return = (_probes, value) => value; - globalThis.$dd_throw = () => {}; +// Fetch the published Browser Debugger SDK bundle and cache it on disk so it can be +// injected into the benchmark page. Fetched fresh per run so the benchmark always tracks +// the SDK that is actually shipping; the bundle only provides the real runtime hooks and +// never participates in the measured hot path. +const ensureDebuggerSdkBundle = async (bundlePath: string): Promise => { + // The bench test process runs with nock.disableNetConnect() (see _playwright/testParams), + // so briefly allow the CDN host, fetch the bundle, then restore the global block. + const sdkHostname = new URL(SDK_BUNDLE_URL).hostname; + nock.enableNetConnect(sdkHostname); + // Hold onto the response headers so we can derive the CDN object's build fingerprint, which + // pins the exact published build behind the (ambiguous) baked SDK version. + let responseHeaders: Headers | undefined; + try { + const bundle = await doRequest({ + url: SDK_BUNDLE_URL, + type: 'text', + retries: 3, + onResponse: (response) => { + responseHeaders = response.headers; + }, + }); + if (!bundle.includes('DD_DEBUGGER')) { + throw new Error(`Fetched Browser Debugger SDK from ${SDK_BUNDLE_URL} looks invalid`); + } + await outputFile(bundlePath, bundle); + } finally { + nock.disableNetConnect(); + } + + // The CDN (S3/CloudFront) always returns these for the bundle object, and the fingerprint is + // required provenance, so a missing header means the URL or infra changed: fail loudly rather + // than record a build we cannot identify (mirroring how the SDK version is asserted). + if (!responseHeaders) { + throw new Error( + `No response captured while fetching the Browser Debugger SDK from ${SDK_BUNDLE_URL}`, + ); + } + const lastModified = responseHeaders.get('last-modified'); + const publishedAt = lastModified ? new Date(lastModified) : undefined; + if (!publishedAt || Number.isNaN(publishedAt.getTime())) { + throw new Error( + `Browser Debugger SDK response is missing a valid Last-Modified header (${SDK_BUNDLE_URL})`, + ); + } + const etag = responseHeaders.get('etag'); + if (!etag) { + throw new Error( + `Browser Debugger SDK response is missing an ETag header (${SDK_BUNDLE_URL})`, + ); + } + return { + publishedAt: publishedAt.toISOString(), + etag: normalizeEtag(etag), + }; +}; + +// Vendor the real Browser Debugger SDK into the page: mock its only dormant egress (the +// probe-delivery poll, matched by path) and inline the CDN bundle so window.DD_DEBUGGER +// exists before any page script. The mocked poll returns an empty probe set, which keeps the +// SDK dormant (no active probes) for the whole benchmark. +const installRealDebuggerSdk = async (page: Page, bundlePath: string) => { + await page.route(`**${DEBUGGER_PROBES_PATH}`, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nextCursor: '', updates: [], deletions: [] }), + }); }); + + // The bundle makes no network calls at load (only init() polls), so inlining it at + // document_start is safe and defines window.DD_DEBUGGER for initRealDebuggerSdk to use. + await page.addInitScript({ path: bundlePath }); +}; + +// Initialize the real SDK dormant (no active probes), wiring the production $dd_probes / +// $dd_entry / $dd_return / $dd_throw. Called after the page has loaded rather than from a +// document_start init script: init() starts the probe-delivery poll, and a fetch fired that +// early escapes Playwright route interception in Firefox (it hits the network and the SDK +// logs a "Delivery API poll error"). Initializing post-load fires the poll while interception +// is reliably active in every browser. `proxyOrigin` is the page origin, keeping the poll +// same-origin (no CORS preflight). The instrumented workloads only run later (in +// runBenchPair), so $dd_probes is the real getProbes for every measured call even though init +// happens after the bundle's bootstrap stub. +const initRealDebuggerSdk = async (page: Page, proxyOrigin: string) => { + await page.evaluate((proxy) => { + globalThis.DD_DEBUGGER?.init({ + clientToken: 'pub00000000000000000000000000000000', + service: 'live-debugger-runtime-bench', + site: 'datadoghq.com', + proxy, + // One day: only the initial poll fires, avoiding re-poll noise mid-measurement. + pollInterval: 24 * 60 * 60 * 1000, + }); + }, proxyOrigin); }; describe('Live Debugger Runtime Benchmark', () => { + // Build fingerprint of the CDN bundle fetched for this run, captured once in beforeAll + // and attached to each test so the reporter can pin the exact measured build. + let sdkBuild: SdkBuild | undefined; + beforeAll(async ({ publicDir, bundlers, suiteName }) => { const source = path.resolve(__dirname, 'project'); const destination = path.resolve(publicDir, suiteName); await buildBenchProject(source, destination, bundlers); + const bundlePath = path.resolve(destination, SDK_BUNDLE_RELATIVE_PATH); + sdkBuild = await ensureDebuggerSdkBundle(bundlePath); }); test('Measures SDK-loaded dormant runtime overhead', async ({ @@ -158,10 +257,12 @@ describe('Live Debugger Runtime Benchmark', () => { browserName, suiteName, devServerUrl, + publicDir, }, testInfo) => { const errors: string[] = []; const projectName = testInfo.project.name; const testBaseUrl = `${devServerUrl}/${suiteName}`; + const bundlePath = path.resolve(publicDir, suiteName, SDK_BUNDLE_RELATIVE_PATH); page.on('pageerror', (error) => errors.push(error.message)); page.on('console', (msg) => { @@ -177,11 +278,43 @@ describe('Live Debugger Runtime Benchmark', () => { } }); - await installDormantDebuggerSdkHooks(page); + // Mock the SDK's probe poll and inline the real CDN bundle before navigation. + await installRealDebuggerSdk(page, bundlePath); const crossOriginIsolated = await userFlow(testBaseUrl, page, bundler); expect(crossOriginIsolated).toBe(true); + // Initialize the SDK now that the page is loaded, then wait for its (mocked) probe + // poll so the registry has settled (empty) before measuring. `devServerUrl` is the + // page origin, keeping the poll same-origin. + const probePoll = page.waitForResponse((response) => + response.url().includes(DEBUGGER_PROBES_PATH), + ); + await initRealDebuggerSdk(page, devServerUrl); + await probePoll; + + // Confirm the real SDK loaded and is dormant (no active probes) before measuring. + const sdkState = await page.evaluate(() => { + const probesForAbsentFunction = globalThis.$dd_probes( + 'live-debugger-runtime-bench;__absent__', + ); + return { + hasSdk: typeof globalThis.DD_DEBUGGER !== 'undefined', + version: globalThis.DD_DEBUGGER?.version, + dormant: probesForAbsentFunction === undefined, + }; + }); + expect(sdkState.hasSdk).toBe(true); + expect(sdkState.dormant).toBe(true); + + // The SDK always exposes its build version (set by makePublicApi). A missing version + // means the SDK failed to load or changed its contract, so fail loudly here rather + // than silently reporting an "unknown" version downstream. + const sdkVersion = sdkState.version; + if (!sdkVersion) { + throw new Error('Browser Debugger SDK did not expose a version'); + } + const results = await page.evaluate((options) => { const bench = globalThis.ddBench; if (!bench) { @@ -216,10 +349,16 @@ describe('Live Debugger Runtime Benchmark', () => { }); }, BENCH_OPTIONS); + if (!sdkBuild) { + throw new Error('Browser Debugger SDK build fingerprint was not captured in beforeAll'); + } + await testInfo.attach('live-debugger-runtime-bench', { body: JSON.stringify( { browserName: projectName || browserName, + sdkVersion, + sdkBuild, results, }, null, diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts index 1348e9dfd..afc4882c1 100644 --- a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts @@ -97,4 +97,32 @@ describe('Live Debugger runtime benchmark reporter', () => { expect(comment).toContain('<= 1.23%'); expect(comment).toContain('overhead upper'); }); + + test('should label the SDK build with publish date and a short S3 ETag', () => { + const row = createBenchResultRow(); + // CloudFront returns a weak ETag (W/"...") when it compresses the response; the + // label must reduce it to the bare content hash, not leak the W/ prefix or quotes. + const comment = renderMarkdownComment([row], [], '7.4.0', { + publishedAt: '2026-06-23T08:01:00.000Z', + etag: 'W/"0766e1c8f7af8eceb34ea84386a51f37"', + }); + + // The build fingerprint disambiguates the (ambiguous) baked version, and the hash is + // explicitly labeled "S3 ETag" so it is not mistaken for a git commit SHA. + expect(comment).toContain( + 'Browser Debugger SDK: `7.4.0` · built 2026-06-23 · S3 ETag `0766e1c8`', + ); + // The ETag is shortened, with no weak-validator prefix, quotes, or full bare hash. + expect(comment).not.toContain('0766e1c8f7af8eceb34ea84386a51f37'); + expect(comment).not.toContain('W/'); + }); + + test('should render the SDK version alone when no build fingerprint is available', () => { + const row = createBenchResultRow(); + const comment = renderMarkdownComment([row], [], '7.4.0'); + + expect(comment).toContain('Browser Debugger SDK: `7.4.0`'); + expect(comment).not.toContain('S3 ETag'); + expect(comment).not.toContain('built '); + }); }); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts index be7982ec9..0cda84f88 100644 --- a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts @@ -14,6 +14,7 @@ import type { RawBenchAttachment, RawVariantResult, RawWorkloadResult, + SdkBuild, } from '../types'; import { @@ -348,13 +349,38 @@ export const buildAlignedTable = (rows: BenchResultRow[]) => { return lines.join('\n'); }; -const printRows = (rows: BenchResultRow[]) => { +// Normalize an HTTP ETag header value to its bare content hash: strip the optional weak +// validator prefix (W/) and the surrounding quotes. CloudFront returns a weak ETag (W/"...") +// when it compresses the response, so the raw header varies in shape for identical content. +export const normalizeEtag = (etag: string): string => etag.replace(/^W\//, '').replace(/"/g, ''); + +// Build the "Browser Debugger SDK" label: the baked version plus the CDN build fingerprint +// (publish date and short S3 ETag). The ETag is explicitly labeled so it is not mistaken for +// a git commit SHA. `code` wraps the version/etag in markdown backticks when rendering a comment. +const formatSdkLabel = ( + sdkVersion: string, + sdkBuild: SdkBuild | undefined, + code: (value: string) => string = (value) => value, +): string => { + const parts = [code(sdkVersion)]; + if (sdkBuild) { + parts.push(`built ${sdkBuild.publishedAt.slice(0, 10)}`); + const shortEtag = normalizeEtag(sdkBuild.etag).slice(0, 8); + parts.push(`S3 ETag ${code(shortEtag)}`); + } + return parts.join(' · '); +}; + +const printRows = (rows: BenchResultRow[], sdkVersion?: string, sdkBuild?: SdkBuild) => { if (rows.length === 0) { console.log('\nLive Debugger runtime benchmark produced no results.'); return; } console.log('\nLive Debugger runtime benchmark'); + if (sdkVersion) { + console.log(`Browser Debugger SDK: ${formatSdkLabel(sdkVersion, sdkBuild)}`); + } console.log(buildAlignedTable(rows)); }; @@ -373,7 +399,12 @@ const printOutputPaths = (resultsFile: string) => { console.log(`\nRaw results written to ${resultsFile}`); }; -export const renderMarkdownComment = (rows: BenchResultRow[], failures: BenchFailure[]) => { +export const renderMarkdownComment = ( + rows: BenchResultRow[], + failures: BenchFailure[], + sdkVersion?: string, + sdkBuild?: SdkBuild, +) => { let body = `${COMMENT_MARKER}\n## Live Debugger Runtime Benchmark\n\n`; if (rows.length === 0) { @@ -381,6 +412,10 @@ export const renderMarkdownComment = (rows: BenchResultRow[], failures: BenchFai } else { body += 'SDK-loaded dormant-probe runtime overhead, measured against an uninstrumented bundle in the same browser session.\n\n'; + if (sdkVersion) { + const sdkLabel = formatSdkLabel(sdkVersion, sdkBuild, (value) => `\`${value}\``); + body += `Browser Debugger SDK: ${sdkLabel}\n\n`; + } body += '| Browser | Workload | Quality | Per-call overhead upper |\n'; body += '| --- | --- | --- | ---: |\n'; @@ -412,6 +447,8 @@ export const renderMarkdownComment = (rows: BenchResultRow[], failures: BenchFai export default class BenchReporter implements Reporter { private rows: BenchResultRow[] = []; private failures: BenchFailure[] = []; + private sdkVersion: string | undefined; + private sdkBuild: SdkBuild | undefined; onTestEnd(test: TestCase, result: TestResult) { if (result.status !== 'passed') { @@ -427,6 +464,8 @@ export default class BenchReporter implements Reporter { const attachments = parseAttachmentBody(result); for (const attachment of attachments) { + this.sdkVersion = attachment.sdkVersion; + this.sdkBuild = attachment.sdkBuild; this.rows.push(...toRows(attachment)); } } @@ -439,9 +478,14 @@ export default class BenchReporter implements Reporter { return aKey.localeCompare(bKey); }); - printRows(this.rows); + printRows(this.rows, this.sdkVersion, this.sdkBuild); printFailures(this.failures); - const markdownComment = renderMarkdownComment(this.rows, this.failures); + const markdownComment = renderMarkdownComment( + this.rows, + this.failures, + this.sdkVersion, + this.sdkBuild, + ); outputFileSync(COMMENT_FILE, markdownComment); const generatedAt = new Date().toISOString(); const resultsFile = buildResultsFilePath(generatedAt); @@ -451,6 +495,8 @@ export default class BenchReporter implements Reporter { { status: result.status, generatedAt, + sdkVersion: this.sdkVersion, + sdkBuild: this.sdkBuild, rows: this.rows, failures: this.failures, }, diff --git a/packages/tests/src/bench/liveDebuggerRuntime/types.ts b/packages/tests/src/bench/liveDebuggerRuntime/types.ts index 43838d949..2db38e69f 100644 --- a/packages/tests/src/bench/liveDebuggerRuntime/types.ts +++ b/packages/tests/src/bench/liveDebuggerRuntime/types.ts @@ -53,8 +53,20 @@ export type BrowserBenchApi = { workloads: BrowserBenchWorkload[]; }; +// Build fingerprint for the fetched CDN bundle. The baked sdkVersion alone is ambiguous +// (the published /v7/ bundle keeps the same release-format version across many builds), so +// we capture the CDN object's publish date and S3 ETag (content hash) to pin the exact +// build that was measured. Required: the benchmark asserts both headers are present when it +// fetches the bundle, mirroring how it asserts the SDK version. +export type SdkBuild = { + publishedAt: string; + etag: string; +}; + export type RawBenchAttachment = { browserName: string; + sdkVersion: string; + sdkBuild: SdkBuild; results: RawWorkloadResult[]; };