diff --git a/packages/perf-harness/README.md b/packages/perf-harness/README.md new file mode 100644 index 000000000..484670a95 --- /dev/null +++ b/packages/perf-harness/README.md @@ -0,0 +1,39 @@ +# Graphile performance harness + +Reusable infrastructure for measuring Graphile schema builds in fresh Node +processes. The core accepts any list of serializable benchmark cases; it does not +interpret case names or optimization-specific configuration. + +Each measurement receives a new PID, starts Node with `--expose-gc`, runs a +deterministic GC sequence, records build time and memory metrics, validates a +runtime query, and reports a schema hash. Cases can opt into schema equivalence +groups and provide their own lifecycle validation through the worker result. + +## Extending the harness + +Define a suite and provide a dedicated worker entry point: + +```ts +const suite = { + name: 'example', + cases: [ + { + name: 'baseline', + workerConfig: { schemas: ['cperf_example'] }, + expectedSchemaGroup: 'example-schema', + }, + ], +}; + +await runBenchmarkSuite(suite, options, workerPath); +``` + +`workerConfig` must be JSON-serializable. Logic is implemented in the worker +entry rather than serializing functions across process boundaries. + +The package includes `stock-worker.js` as a minimal upstream Graphile baseline. +Database credentials are supplied through `CPERF_DATABASE_URL` and are redacted +from worker failures and JSON reports. + +The PostgreSQL fixture command only creates a previously absent schema whose +name starts with `cperf_`; it never drops or replaces schemas. diff --git a/packages/perf-harness/__tests__/fixture.test.ts b/packages/perf-harness/__tests__/fixture.test.ts new file mode 100644 index 000000000..08e9109e1 --- /dev/null +++ b/packages/perf-harness/__tests__/fixture.test.ts @@ -0,0 +1,22 @@ +import { + validateFixtureSchema, + validateFixtureTableCount, +} from '../src/fixture'; + +describe('fixture safety', () => { + test('only accepts narrowly scoped benchmark schema names', () => { + expect(validateFixtureSchema('cperf_example_1')).toBe('cperf_example_1'); + expect(() => validateFixtureSchema('public')).toThrow( + 'must start with cperf_' + ); + expect(() => + validateFixtureSchema('cperf_example; drop schema public') + ).toThrow('must start with cperf_'); + }); + + test('bounds generated fixture size', () => { + expect(validateFixtureTableCount(64)).toBe(64); + expect(() => validateFixtureTableCount(0)).toThrow('between 1 and 500'); + expect(() => validateFixtureTableCount(501)).toThrow('between 1 and 500'); + }); +}); diff --git a/packages/perf-harness/__tests__/fixtures/fake-worker.js b/packages/perf-harness/__tests__/fixtures/fake-worker.js new file mode 100644 index 000000000..ea69962ac --- /dev/null +++ b/packages/perf-harness/__tests__/fixtures/fake-worker.js @@ -0,0 +1,30 @@ +'use strict'; + +const envelope = JSON.parse( + Buffer.from(process.env.CPERF_WORKER_CONFIG, 'base64url').toString('utf8') +); +const value = envelope.workerConfig.value; +const memory = { + rss: value, + heapTotal: value, + heapUsed: value, + external: value, + arrayBuffers: value, +}; +const result = { + status: 'ok', + pid: process.pid, + caseName: envelope.caseName, + buildMs: value, + schemaHash: envelope.workerConfig.schemaHash, + schemaTypeCount: 10, + runtimeVerified: true, + caseValidation: { passed: true, errors: [] }, + memory: { + baseline: memory, + afterBuild: memory, + delta: memory, + processPeakRss: value, + }, +}; +process.stdout.write(`CPERF_RESULT ${JSON.stringify(result)}\n`); diff --git a/packages/perf-harness/__tests__/process.test.ts b/packages/perf-harness/__tests__/process.test.ts new file mode 100644 index 000000000..65fd7ef64 --- /dev/null +++ b/packages/perf-harness/__tests__/process.test.ts @@ -0,0 +1,22 @@ +import { resolve } from 'node:path'; + +import { runWorkerProcess } from '../src/process'; + +describe('fresh worker process', () => { + test('uses distinct PIDs and does not expose the database URL', async () => { + const worker = resolve(__dirname, 'fixtures/fake-worker.js'); + const definition = { + name: 'baseline', + workerConfig: { value: 1, schemaHash: 'same' }, + }; + const databaseUrl = 'postgres://secret@example.test/database'; + const first = await runWorkerProcess(worker, databaseUrl, definition); + const second = await runWorkerProcess(worker, databaseUrl, definition); + expect(first.pid).not.toBe(process.pid); + expect(second.pid).not.toBe(process.pid); + expect(first.pid).not.toBe(second.pid); + expect(JSON.stringify([first.result, second.result])).not.toContain( + databaseUrl + ); + }); +}); diff --git a/packages/perf-harness/__tests__/report.test.ts b/packages/perf-harness/__tests__/report.test.ts new file mode 100644 index 000000000..6d7e043d7 --- /dev/null +++ b/packages/perf-harness/__tests__/report.test.ts @@ -0,0 +1,78 @@ +import { + compareCases, + summarizeCase, + validateSchemaGroups, +} from '../src/report'; +import type { BenchmarkRun, SuccessfulWorkerResult } from '../src/types'; + +const result = (caseName: string, value: number): SuccessfulWorkerResult => ({ + status: 'ok', + pid: value, + caseName, + buildMs: value, + schemaHash: 'same', + schemaTypeCount: 10, + runtimeVerified: true, + caseValidation: { passed: true, errors: [] }, + memory: { + baseline: { + rss: 10, + heapTotal: 10, + heapUsed: 10, + external: 10, + arrayBuffers: 10, + }, + afterBuild: { + rss: value, + heapTotal: value, + heapUsed: value, + external: value, + arrayBuffers: value, + }, + delta: { + rss: value - 10, + heapTotal: value - 10, + heapUsed: value - 10, + external: value - 10, + arrayBuffers: value - 10, + }, + processPeakRss: value, + }, +}); + +describe('generic reports', () => { + test('summarizes arbitrary cases and compares medians', () => { + const runs: BenchmarkRun[] = [10, 30, 20].map((value, index) => ({ + repetition: index + 1, + position: 1, + caseName: 'base', + result: result('base', value), + })); + runs.push( + ...[5, 15, 10].map((value, index) => ({ + repetition: index + 1, + position: 2, + caseName: 'candidate', + result: result('candidate', value), + })) + ); + const base = summarizeCase(runs, 'base')!; + const candidate = summarizeCase(runs, 'candidate')!; + expect( + compareCases('base', 'candidate', base, candidate).buildMs.percentChange + ).toBe(-50); + expect( + validateSchemaGroups( + [ + { name: 'base', workerConfig: null, expectedSchemaGroup: 'schema' }, + { + name: 'candidate', + workerConfig: null, + expectedSchemaGroup: 'schema', + }, + ], + runs + ) + ).toEqual({ equivalent: true, hashes: { schema: 'same' }, errors: [] }); + }); +}); diff --git a/packages/perf-harness/__tests__/run.test.ts b/packages/perf-harness/__tests__/run.test.ts new file mode 100644 index 000000000..0628e1ba4 --- /dev/null +++ b/packages/perf-harness/__tests__/run.test.ts @@ -0,0 +1,37 @@ +import { resolve } from 'node:path'; + +import { runBenchmarkSuite } from '../src/run'; + +describe('generic suite runner', () => { + test('validates fresh processes and schema groups without fixed case names', async () => { + const report = await runBenchmarkSuite( + { + name: 'test-suite', + cases: ['alpha', 'beta', 'gamma'].map((name, index) => ({ + name, + workerConfig: { value: index + 1, schemaHash: 'same' }, + expectedSchemaGroup: 'schema', + })), + }, + { + databaseUrl: 'postgres:///not-used-by-fake-worker', + repetitions: 1, + seed: 1, + order: ['alpha', 'beta', 'gamma'], + }, + resolve(__dirname, 'fixtures/fake-worker.js') + ); + expect(report.validation).toEqual( + expect.objectContaining({ + allRunsSucceeded: true, + freshProcessPerRun: true, + caseValidationPassed: true, + schemaGroupsEquivalent: true, + schemaGroups: { schema: 'same' }, + errors: [], + }) + ); + expect(new Set(report.runs.map((run) => run.result.pid)).size).toBe(3); + expect(JSON.stringify(report)).not.toContain('postgres:///'); + }); +}); diff --git a/packages/perf-harness/__tests__/schedule.test.ts b/packages/perf-harness/__tests__/schedule.test.ts new file mode 100644 index 000000000..89d5e672f --- /dev/null +++ b/packages/perf-harness/__tests__/schedule.test.ts @@ -0,0 +1,29 @@ +import { makeSchedule } from '../src/schedule'; + +const cases = [ + { name: 'a', workerConfig: null }, + { name: 'b', workerConfig: null }, + { name: 'c', workerConfig: null }, +]; + +describe('generic benchmark scheduling', () => { + test('is deterministic and supports any case list', () => { + const first = makeSchedule(cases, 4, 1234); + expect(makeSchedule(cases, 4, 1234)).toEqual(first); + expect(makeSchedule(cases, 4, 4321)).not.toEqual(first); + for (let repetition = 1; repetition <= 4; repetition += 1) { + expect( + first + .filter((item) => item.repetition === repetition) + .map((item) => item.caseName) + .sort() + ).toEqual(['a', 'b', 'c']); + } + }); + + test('accepts an exact order containing each case once', () => { + expect( + makeSchedule(cases, 2, 1, ['c', 'a', 'b']).map((item) => item.caseName) + ).toEqual(['c', 'a', 'b', 'c', 'a', 'b']); + }); +}); diff --git a/packages/perf-harness/jest.config.js b/packages/perf-harness/jest.config.js new file mode 100644 index 000000000..f34711d4e --- /dev/null +++ b/packages/perf-harness/jest.config.js @@ -0,0 +1,12 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }], + }, + testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + testPathIgnorePatterns: ['/__tests__/fixtures/'], +}; diff --git a/packages/perf-harness/package.json b/packages/perf-harness/package.json new file mode 100644 index 000000000..8d23b8bc1 --- /dev/null +++ b/packages/perf-harness/package.json @@ -0,0 +1,36 @@ +{ + "name": "@constructive-io/perf-harness", + "version": "0.1.0", + "private": true, + "description": "Reusable fresh-process Graphile performance harness", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "bin": { + "cperf": "index.js" + }, + "scripts": { + "clean": "makage clean", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest" + }, + "dependencies": { + "graphile-build": "5.1.1", + "graphile-build-pg": "5.1.3", + "graphile-config": "1.1.0", + "graphql": "16.13.0", + "pg": "^8.21.0", + "postgraphile": "5.1.4" + }, + "devDependencies": { + "@types/node": "^22.19.11", + "@types/pg": "^8.20.4", + "makage": "^0.3.0" + }, + "engines": { + "node": ">=22" + }, + "license": "MIT" +} diff --git a/packages/perf-harness/src/fixture.ts b/packages/perf-harness/src/fixture.ts new file mode 100644 index 000000000..b7f35935b --- /dev/null +++ b/packages/perf-harness/src/fixture.ts @@ -0,0 +1,127 @@ +import { Pool } from 'pg'; + +export const FIXTURE_VERSION = 1; + +export interface PrepareFixtureOptions { + databaseUrl: string; + schema: string; + tables: number; +} + +export interface PreparedFixture { + fixtureVersion: number; + database: string; + serverVersion: string; + schema: string; + tableCount: number; + functionCount: number; +} + +export const validateFixtureSchema = (schema: string): string => { + if ( + !/^cperf_[a-z0-9_]*$/.test(schema) || + schema.length > 63 || + schema.includes('\0') + ) { + throw new Error( + 'fixture schema must start with cperf_, use only lowercase letters, digits, and underscores, and fit PostgreSQL identifiers' + ); + } + return schema; +}; + +export const validateFixtureTableCount = (tables: number): number => { + if (!Number.isSafeInteger(tables) || tables < 1 || tables > 500) { + throw new Error('fixture table count must be an integer between 1 and 500'); + } + return tables; +}; + +const quoteIdentifier = (identifier: string): string => + `"${identifier.replaceAll('"', '""')}"`; + +export const prepareFixture = async ( + options: PrepareFixtureOptions +): Promise => { + const schema = validateFixtureSchema(options.schema); + const tables = validateFixtureTableCount(options.tables); + const quotedSchema = quoteIdentifier(schema); + const pool = new Pool({ connectionString: options.databaseUrl, max: 1 }); + const client = await pool.connect(); + try { + await client.query('begin'); + const existing = await client.query<{ exists: boolean }>( + 'select exists(select 1 from pg_catalog.pg_namespace where nspname = $1) as exists', + [schema] + ); + if (existing.rows[0]?.exists) { + throw new Error( + `fixture schema '${schema}' already exists; this command never replaces schemas` + ); + } + await client.query(`create schema ${quotedSchema}`); + await client.query( + `comment on schema ${quotedSchema} is 'cperf fixture version ${FIXTURE_VERSION}'` + ); + await client.query( + `create type ${quotedSchema}."entity_status" as enum ('draft', 'active', 'archived')` + ); + await client.query(` + create table ${quotedSchema}."account" ( + id bigint generated always as identity primary key, + external_id uuid not null unique, + name text not null, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() + ) + `); + for (let index = 1; index <= tables; index += 1) { + const table = quoteIdentifier(`entity_${index}`); + const functionName = quoteIdentifier(`entity_${index}_by_account`); + const indexName = quoteIdentifier(`entity_${index}_account_created_idx`); + await client.query(` + create table ${quotedSchema}.${table} ( + id bigint generated always as identity primary key, + account_id bigint not null references ${quotedSchema}."account"(id), + status ${quotedSchema}."entity_status" not null default 'draft', + title text not null, + tags text[] not null default array[]::text[], + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + unique (account_id, title) + ); + create index ${indexName} + on ${quotedSchema}.${table} (account_id, created_at desc); + create function ${quotedSchema}.${functionName}(requested_account_id bigint) + returns setof ${quotedSchema}.${table} + language sql stable + as 'select * from ${quotedSchema}.${table} where account_id = requested_account_id'; + `); + } + const identity = await client.query<{ + database: string; + server_version: string; + }>( + "select current_database() as database, current_setting('server_version') as server_version" + ); + await client.query('commit'); + return { + fixtureVersion: FIXTURE_VERSION, + database: identity.rows[0].database, + serverVersion: identity.rows[0].server_version, + schema, + tableCount: tables + 1, + functionCount: tables, + }; + } catch (error) { + try { + await client.query('rollback'); + } catch { + // Preserve the fixture preparation error; the client is discarded below. + } + throw error; + } finally { + client.release(); + await pool.end(); + } +}; diff --git a/packages/perf-harness/src/index.ts b/packages/perf-harness/src/index.ts new file mode 100644 index 000000000..7829d7dc6 --- /dev/null +++ b/packages/perf-harness/src/index.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +export * from './fixture'; +export * from './metrics'; +export * from './process'; +export * from './report'; +export * from './run'; +export * from './schedule'; +export * from './types'; + +import { cliMain } from './run'; + +if (require.main === module) { + void cliMain().catch((error: unknown) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n` + ); + process.exitCode = 1; + }); +} diff --git a/packages/perf-harness/src/metrics.ts b/packages/perf-harness/src/metrics.ts new file mode 100644 index 000000000..007e79fa1 --- /dev/null +++ b/packages/perf-harness/src/metrics.ts @@ -0,0 +1,79 @@ +import { performance } from 'node:perf_hooks'; + +import type { + CaseValidation, + JsonValue, + MemorySnapshot, + SuccessfulWorkerResult, +} from './types'; + +export interface MeasuredCaseResult { + schemaHash: string; + schemaTypeCount: number; + runtimeVerified: true; + caseValidation?: CaseValidation; + metadata?: Record; +} + +const collectGarbage = (): void => { + if (typeof global.gc !== 'function') { + throw new Error('benchmark worker requires Node --expose-gc'); + } + global.gc(); + global.gc(); + global.gc(); +}; + +const memorySnapshot = (): MemorySnapshot => { + const memory = process.memoryUsage(); + return { + rss: memory.rss, + heapTotal: memory.heapTotal, + heapUsed: memory.heapUsed, + external: memory.external, + arrayBuffers: memory.arrayBuffers, + }; +}; + +const memoryDelta = ( + baseline: MemorySnapshot, + afterBuild: MemorySnapshot +): MemorySnapshot => ({ + rss: afterBuild.rss - baseline.rss, + heapTotal: afterBuild.heapTotal - baseline.heapTotal, + heapUsed: afterBuild.heapUsed - baseline.heapUsed, + external: afterBuild.external - baseline.external, + arrayBuffers: afterBuild.arrayBuffers - baseline.arrayBuffers, +}); + +export const measureBenchmarkCase = async ( + caseName: string, + build: () => Promise, + validate: (built: Built) => Promise +): Promise => { + collectGarbage(); + const baseline = memorySnapshot(); + const startedAt = performance.now(); + const built = await build(); + const buildMs = performance.now() - startedAt; + const measured = await validate(built); + collectGarbage(); + const afterBuild = memorySnapshot(); + return { + status: 'ok', + pid: process.pid, + caseName, + buildMs, + schemaHash: measured.schemaHash, + schemaTypeCount: measured.schemaTypeCount, + runtimeVerified: measured.runtimeVerified, + caseValidation: measured.caseValidation ?? { passed: true, errors: [] }, + ...(measured.metadata ? { metadata: measured.metadata } : {}), + memory: { + baseline, + afterBuild, + delta: memoryDelta(baseline, afterBuild), + processPeakRss: process.resourceUsage().maxRSS * 1024, + }, + }; +}; diff --git a/packages/perf-harness/src/process.ts b/packages/perf-harness/src/process.ts new file mode 100644 index 000000000..14c5464a6 --- /dev/null +++ b/packages/perf-harness/src/process.ts @@ -0,0 +1,130 @@ +import { spawn } from 'node:child_process'; + +import type { + BenchmarkCaseDefinition, + WorkerConfigEnvelope, + WorkerResult, +} from './types'; + +export const WORKER_RESULT_PREFIX = 'CPERF_RESULT '; +export const DATABASE_URL_ENV = 'CPERF_DATABASE_URL'; +export const WORKER_CONFIG_ENV = 'CPERF_WORKER_CONFIG'; + +export interface SpawnedWorkerResult { + pid: number; + result: WorkerResult; +} + +const lastLines = (value: string, count = 20): string => + value.trim().split('\n').slice(-count).join('\n'); + +export const redactSecret = (value: string, secret: string): string => + secret ? value.replaceAll(secret, '') : value; + +export const runWorkerProcess = ( + workerPath: string, + databaseUrl: string, + definition: BenchmarkCaseDefinition +): Promise => + new Promise((resolve, reject) => { + const config: WorkerConfigEnvelope = { + caseName: definition.name, + workerConfig: definition.workerConfig, + }; + const child = spawn(process.execPath, ['--expose-gc', workerPath], { + env: { + ...process.env, + NODE_ENV: 'production', + GRAPHILE_ENV: 'production', + [DATABASE_URL_ENV]: databaseUrl, + [WORKER_CONFIG_ENV]: Buffer.from(JSON.stringify(config)).toString( + 'base64url' + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const pid = child.pid; + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + child.once('error', reject); + child.once('close', (code, signal) => { + const resultLine = stdout + .split('\n') + .reverse() + .find((line) => line.startsWith(WORKER_RESULT_PREFIX)); + if (!resultLine) { + reject( + new Error( + redactSecret( + `benchmark worker ${pid ?? 'unknown'} exited without a result ` + + `(code=${String(code)}, signal=${String(signal)})` + + (stderr.trim() ? `\n${lastLines(stderr)}` : ''), + databaseUrl + ) + ) + ); + return; + } + try { + const result = JSON.parse( + resultLine.slice(WORKER_RESULT_PREFIX.length) + ) as WorkerResult; + if (typeof pid !== 'number' || result.pid !== pid) { + throw new Error( + `worker PID mismatch: spawned ${String(pid)}, reported ${String( + result.pid + )}` + ); + } + if (result.caseName !== definition.name) { + throw new Error( + `worker case mismatch: expected ${definition.name}, reported ${result.caseName}` + ); + } + if (result.status === 'ok' && code !== 0) { + throw new Error(`successful worker exited with code ${String(code)}`); + } + resolve({ pid, result }); + } catch (error) { + reject( + new Error( + redactSecret( + `invalid result from benchmark worker ${String(pid)}: ${String( + error instanceof Error ? error.message : error + )}`, + databaseUrl + ) + ) + ); + } + }); + }); + +export const parseWorkerEnvelope = ( + encoded: string | undefined +): WorkerConfigEnvelope => { + if (!encoded) throw new Error(`${WORKER_CONFIG_ENV} is required`); + const parsed = JSON.parse( + Buffer.from(encoded, 'base64url').toString('utf8') + ) as Partial; + if ( + typeof parsed.caseName !== 'string' || + parsed.caseName.length === 0 || + parsed.workerConfig === undefined + ) { + throw new Error('worker configuration envelope is invalid'); + } + return parsed as WorkerConfigEnvelope; +}; + +export const writeWorkerResult = (result: WorkerResult): void => { + process.stdout.write(`${WORKER_RESULT_PREFIX}${JSON.stringify(result)}\n`); +}; diff --git a/packages/perf-harness/src/report.ts b/packages/perf-harness/src/report.ts new file mode 100644 index 000000000..680852a96 --- /dev/null +++ b/packages/perf-harness/src/report.ts @@ -0,0 +1,123 @@ +import type { + BenchmarkCaseDefinition, + BenchmarkRun, + CaseComparison, + CaseSummary, + MetricComparison, + MetricSummary, + SuccessfulWorkerResult, +} from './types'; + +const median = (values: readonly number[]): number => { + if (values.length === 0) throw new Error('cannot summarize zero values'); + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +}; + +const metricSummary = (values: number[]): MetricSummary => ({ + median: median(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values, +}); + +const successfulResultsFor = ( + runs: readonly BenchmarkRun[], + caseName: string +): SuccessfulWorkerResult[] => + runs + .filter((run) => run.caseName === caseName && run.result.status === 'ok') + .map((run) => run.result as SuccessfulWorkerResult); + +export const summarizeCase = ( + runs: readonly BenchmarkRun[], + caseName: string +): CaseSummary | undefined => { + const results = successfulResultsFor(runs, caseName); + if (results.length === 0) return undefined; + return { + sampleCount: results.length, + buildMs: metricSummary(results.map((result) => result.buildMs)), + heapUsedAfterBuild: metricSummary( + results.map((result) => result.memory.afterBuild.heapUsed) + ), + heapUsedDelta: metricSummary( + results.map((result) => result.memory.delta.heapUsed) + ), + rssAfterBuild: metricSummary( + results.map((result) => result.memory.afterBuild.rss) + ), + rssDelta: metricSummary(results.map((result) => result.memory.delta.rss)), + processPeakRss: metricSummary( + results.map((result) => result.memory.processPeakRss) + ), + }; +}; + +const compareMetric = ( + baseline: MetricSummary, + candidate: MetricSummary +): MetricComparison => ({ + baseline: baseline.median, + candidate: candidate.median, + difference: candidate.median - baseline.median, + percentChange: + baseline.median === 0 + ? null + : ((candidate.median - baseline.median) / Math.abs(baseline.median)) * + 100, +}); + +export const compareCases = ( + baselineCase: string, + candidateCase: string, + baseline: CaseSummary, + candidate: CaseSummary +): CaseComparison => ({ + baselineCase, + candidateCase, + buildMs: compareMetric(baseline.buildMs, candidate.buildMs), + heapUsedAfterBuild: compareMetric( + baseline.heapUsedAfterBuild, + candidate.heapUsedAfterBuild + ), + heapUsedDelta: compareMetric(baseline.heapUsedDelta, candidate.heapUsedDelta), + rssAfterBuild: compareMetric(baseline.rssAfterBuild, candidate.rssAfterBuild), + rssDelta: compareMetric(baseline.rssDelta, candidate.rssDelta), + processPeakRss: compareMetric( + baseline.processPeakRss, + candidate.processPeakRss + ), +}); + +export const validateSchemaGroups = ( + definitions: readonly BenchmarkCaseDefinition[], + runs: readonly BenchmarkRun[] +): { + equivalent: boolean; + hashes: Record; + errors: string[]; +} => { + const groups = new Map>(); + for (const definition of definitions) { + if (!definition.expectedSchemaGroup) continue; + const hashes = + groups.get(definition.expectedSchemaGroup) ?? new Set(); + for (const result of successfulResultsFor(runs, definition.name)) { + hashes.add(result.schemaHash); + } + groups.set(definition.expectedSchemaGroup, hashes); + } + const output: Record = {}; + const errors: string[] = []; + for (const [group, hashes] of groups) { + output[group] = hashes.size === 1 ? [...hashes][0] : null; + if (hashes.size !== 1) { + errors.push(`schema group '${group}' did not produce one schema hash`); + } + } + return { equivalent: errors.length === 0, hashes: output, errors }; +}; diff --git a/packages/perf-harness/src/run.ts b/packages/perf-harness/src/run.ts new file mode 100644 index 000000000..d1b889a23 --- /dev/null +++ b/packages/perf-harness/src/run.ts @@ -0,0 +1,274 @@ +import { mkdir, rename, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; + +import { prepareFixture } from './fixture'; +import { DATABASE_URL_ENV, redactSecret, runWorkerProcess } from './process'; +import { summarizeCase, validateSchemaGroups } from './report'; +import { makeSchedule, validateCaseDefinitions } from './schedule'; +import type { + BenchmarkCaseDefinition, + BenchmarkReport, + BenchmarkRun, + BenchmarkSuiteDefinition, +} from './types'; + +export interface RunSuiteOptions { + databaseUrl: string; + repetitions: number; + seed: number; + order: string[] | null; + output?: string; +} + +export const runBenchmarkSuite = async ( + suite: BenchmarkSuiteDefinition, + options: RunSuiteOptions, + workerPath: string +): Promise => { + validateCaseDefinitions(suite.cases); + const byName = new Map( + suite.cases.map((definition) => [definition.name, definition]) + ); + const schedule = makeSchedule( + suite.cases, + options.repetitions, + options.seed, + options.order + ); + const runs: BenchmarkRun[] = []; + for (const coordinate of schedule) { + const definition = byName.get(coordinate.caseName)!; + process.stderr.write( + `[${runs.length + 1}/${schedule.length}] repetition ${ + coordinate.repetition + }, ${coordinate.caseName}\n` + ); + try { + const spawned = await runWorkerProcess( + workerPath, + options.databaseUrl, + definition + ); + runs.push({ ...coordinate, result: spawned.result }); + } catch (error) { + runs.push({ + ...coordinate, + result: { + status: 'error', + pid: -1, + caseName: coordinate.caseName, + error: redactSecret( + error instanceof Error ? error.message : String(error), + options.databaseUrl + ), + }, + }); + } + } + const successfulRuns = runs.filter((run) => run.result.status === 'ok'); + const allRunsSucceeded = successfulRuns.length === schedule.length; + const pids = successfulRuns.map((run) => run.result.pid); + const freshProcessPerRun = + allRunsSucceeded && + pids.every((pid) => pid > 0 && pid !== process.pid) && + new Set(pids).size === pids.length; + const caseValidationPassed = + allRunsSucceeded && + successfulRuns.every( + (run) => run.result.status === 'ok' && run.result.caseValidation.passed + ); + const schemaGroups = validateSchemaGroups(suite.cases, runs); + const errors: string[] = []; + for (const run of runs) { + if (run.result.status === 'error') { + errors.push( + `${run.caseName} repetition ${run.repetition}: ${run.result.error}` + ); + } else { + for (const error of run.result.caseValidation.errors) { + errors.push(`${run.caseName} repetition ${run.repetition}: ${error}`); + } + } + } + if (!freshProcessPerRun) { + errors.push('fresh-process validation did not pass for every run'); + } + errors.push(...schemaGroups.errors); + const summaries: Record< + string, + NonNullable> + > = {}; + for (const definition of suite.cases) { + const summary = summarizeCase(runs, definition.name); + if (summary) summaries[definition.name] = summary; + } + return { + format: 'constructive-performance-suite/v1', + generatedAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + architecture: process.arch, + suite, + config: { + repetitions: options.repetitions, + seed: options.seed, + order: options.order, + }, + schedule, + runs, + validation: { + allRunsSucceeded, + freshProcessPerRun, + caseValidationPassed, + schemaGroupsEquivalent: schemaGroups.equivalent, + schemaGroups: schemaGroups.hashes, + errors, + }, + summaries, + }; +}; + +export const writeJsonAtomically = async ( + output: string, + value: unknown +): Promise => { + const absoluteOutput = resolve(output); + await mkdir(dirname(absoluteOutput), { recursive: true }); + const temporary = `${absoluteOutput}.tmp-${process.pid}`; + await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + mode: 0o600, + }); + await rename(temporary, absoluteOutput); + return absoluteOutput; +}; + +interface ParsedArgs { + values: Map; +} + +const parseArgs = (args: readonly string[]): ParsedArgs => { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if ( + !flag?.startsWith('--') || + value === undefined || + value.startsWith('--') + ) { + throw new Error(`expected --name value near '${flag ?? ''}'`); + } + const name = flag.slice(2); + if (values.has(name)) + throw new Error(`--${name} may only be specified once`); + values.set(name, value); + } + return { values }; +}; + +const positiveInteger = ( + value: string | undefined, + name: string, + defaultValue: number, + maximum: number +): number => { + if (value === undefined) return defaultValue; + if (!/^\d+$/.test(value)) throw new Error(`--${name} must be an integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > maximum) { + throw new Error(`--${name} must be between 1 and ${maximum}`); + } + return parsed; +}; + +const databaseUrl = (args: ParsedArgs): string => { + const value = + args.values.get('database-url') ?? process.env[DATABASE_URL_ENV]; + if (!value) { + throw new Error( + `--database-url or the ${DATABASE_URL_ENV} environment variable is required` + ); + } + return value; +}; + +const stringList = (value: string | undefined): string[] | null => { + if (value === undefined) return null; + const result = value.split(','); + if ( + result.some((item) => item.length === 0 || item.trim() !== item) || + new Set(result).size !== result.length + ) { + throw new Error('list values must be unique exact non-empty strings'); + } + return result; +}; + +const parseCases = (encoded: string): BenchmarkCaseDefinition[] => { + const parsed = JSON.parse(Buffer.from(encoded, 'base64url').toString('utf8')); + if (!Array.isArray(parsed)) + throw new Error('--cases must encode a JSON array'); + return parsed as BenchmarkCaseDefinition[]; +}; + +export const cliMain = async (args = process.argv.slice(2)): Promise => { + const [command, ...rest] = args; + const parsed = parseArgs(rest); + if (command === 'prepare') { + const schema = parsed.values.get('schema'); + if (!schema) throw new Error('--schema is required'); + const result = await prepareFixture({ + databaseUrl: databaseUrl(parsed), + schema, + tables: positiveInteger(parsed.values.get('tables'), 'tables', 64, 500), + }); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + if (command !== 'run') { + throw new Error('expected prepare or run command'); + } + const cases = parsed.values.get('cases'); + const worker = parsed.values.get('worker'); + if (!cases || !worker) throw new Error('--cases and --worker are required'); + const report = await runBenchmarkSuite( + { + name: parsed.values.get('suite') ?? 'benchmark-suite', + cases: parseCases(cases), + }, + { + databaseUrl: databaseUrl(parsed), + repetitions: positiveInteger( + parsed.values.get('repetitions'), + 'repetitions', + 3, + 50 + ), + seed: positiveInteger( + parsed.values.get('seed'), + 'seed', + 20260813, + 0xffffffff + ), + order: stringList(parsed.values.get('order')), + output: parsed.values.get('output'), + }, + resolve(worker) + ); + const output = await writeJsonAtomically( + parsed.values.get('output') ?? 'performance-report.json', + report + ); + process.stdout.write( + `${JSON.stringify({ output, validation: report.validation })}\n` + ); + if ( + !report.validation.allRunsSucceeded || + !report.validation.freshProcessPerRun || + !report.validation.caseValidationPassed || + !report.validation.schemaGroupsEquivalent + ) { + process.exitCode = 1; + } +}; diff --git a/packages/perf-harness/src/schedule.ts b/packages/perf-harness/src/schedule.ts new file mode 100644 index 000000000..8eaa98fb8 --- /dev/null +++ b/packages/perf-harness/src/schedule.ts @@ -0,0 +1,77 @@ +import type { BenchmarkCaseDefinition, BenchmarkCoordinate } from './types'; + +const seededRandom = (seed: number): (() => number) => { + let state = seed >>> 0; + return () => { + state += 0x6d2b79f5; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +}; + +const shuffledCaseNames = ( + definitions: readonly BenchmarkCaseDefinition[], + seed: number, + repetition: number +): string[] => { + const result = definitions.map(({ name }) => name); + const random = seededRandom((seed ^ Math.imul(repetition, 0x9e3779b1)) >>> 0); + for (let index = result.length - 1; index > 0; index -= 1) { + const swapIndex = Math.floor(random() * (index + 1)); + [result[index], result[swapIndex]] = [result[swapIndex], result[index]]; + } + return result; +}; + +export const validateCaseDefinitions = ( + definitions: readonly BenchmarkCaseDefinition[] +): void => { + if (definitions.length === 0) { + throw new Error('benchmark suite must contain at least one case'); + } + const names = definitions.map(({ name }) => name); + if ( + names.some( + (name) => name.length === 0 || name.trim() !== name || name.includes('\0') + ) || + new Set(names).size !== names.length + ) { + throw new Error( + 'benchmark case names must be unique exact non-empty strings' + ); + } +}; + +export const makeSchedule = ( + definitions: readonly BenchmarkCaseDefinition[], + repetitions: number, + seed: number, + exactOrder: readonly string[] | null = null +): BenchmarkCoordinate[] => { + validateCaseDefinitions(definitions); + if (!Number.isSafeInteger(repetitions) || repetitions < 1) { + throw new Error('repetitions must be a positive safe integer'); + } + const expectedNames = definitions.map(({ name }) => name).sort(); + const schedule: BenchmarkCoordinate[] = []; + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + const order = exactOrder + ? [...exactOrder] + : shuffledCaseNames(definitions, seed, repetition); + if ( + order.length !== definitions.length || + new Set(order).size !== definitions.length || + [...order].sort().some((name, index) => name !== expectedNames[index]) + ) { + throw new Error( + 'exact order must contain each benchmark case exactly once' + ); + } + order.forEach((caseName, index) => { + schedule.push({ repetition, position: index + 1, caseName }); + }); + } + return schedule; +}; diff --git a/packages/perf-harness/src/stock-worker.ts b/packages/perf-harness/src/stock-worker.ts new file mode 100644 index 000000000..71eb1f357 --- /dev/null +++ b/packages/perf-harness/src/stock-worker.ts @@ -0,0 +1,94 @@ +import { createHash } from 'node:crypto'; + +import { + defaultPreset as graphileBuildPreset, + makeSchema, +} from 'graphile-build'; +import { defaultPreset as graphileBuildPgPreset } from 'graphile-build-pg'; +import { execute, lexicographicSortSchema, parse, printSchema } from 'graphql'; +import { makePgService } from 'postgraphile/adaptors/pg'; + +import { measureBenchmarkCase } from './metrics'; +import { + DATABASE_URL_ENV, + parseWorkerEnvelope, + redactSecret, + WORKER_CONFIG_ENV, + writeWorkerResult, +} from './process'; + +interface StockConfig { + schemas: string[]; +} + +const validateConfig = (value: unknown): StockConfig => { + const schemas = (value as Partial)?.schemas; + if ( + !Array.isArray(schemas) || + schemas.length === 0 || + schemas.some((schema) => typeof schema !== 'string' || schema.length === 0) + ) { + throw new Error('stock worker requires a non-empty schemas array'); + } + return { schemas }; +}; + +const main = async (): Promise => { + const databaseUrl = process.env[DATABASE_URL_ENV] ?? ''; + let caseName = 'unknown'; + try { + if (!databaseUrl) throw new Error(`${DATABASE_URL_ENV} is required`); + const envelope = parseWorkerEnvelope(process.env[WORKER_CONFIG_ENV]); + caseName = envelope.caseName; + const config = validateConfig(envelope.workerConfig); + const service = makePgService({ + connectionString: databaseUrl, + schemas: config.schemas, + pubsub: false, + }); + try { + const result = await measureBenchmarkCase( + caseName, + async () => + makeSchema({ + extends: [graphileBuildPreset, graphileBuildPgPreset], + pgServices: [service], + }), + async ({ schema }) => { + const execution = await execute({ + schema, + document: parse('{ __typename }'), + }); + if ( + execution.errors?.length || + execution.data?.__typename !== 'Query' + ) { + throw new Error('runtime verification query failed'); + } + const schemaText = printSchema(lexicographicSortSchema(schema)); + return { + schemaHash: createHash('sha256').update(schemaText).digest('hex'), + schemaTypeCount: Object.keys(schema.getTypeMap()).length, + runtimeVerified: true as const, + }; + } + ); + writeWorkerResult(result); + } finally { + await service.release(); + } + } catch (error) { + writeWorkerResult({ + status: 'error', + pid: process.pid, + caseName, + error: redactSecret( + error instanceof Error ? error.message : String(error), + databaseUrl + ), + }); + process.exitCode = 1; + } +}; + +if (require.main === module) void main(); diff --git a/packages/perf-harness/src/types.ts b/packages/perf-harness/src/types.ts new file mode 100644 index 000000000..3713a4edc --- /dev/null +++ b/packages/perf-harness/src/types.ts @@ -0,0 +1,129 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = + JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +export interface BenchmarkCaseDefinition { + name: string; + workerConfig: JsonValue; + expectedSchemaGroup?: string; +} + +export interface BenchmarkSuiteDefinition { + name: string; + cases: BenchmarkCaseDefinition[]; +} + +export interface BenchmarkCoordinate { + repetition: number; + position: number; + caseName: string; +} + +export interface WorkerConfigEnvelope { + caseName: string; + workerConfig: JsonValue; +} + +export interface MemorySnapshot { + rss: number; + heapTotal: number; + heapUsed: number; + external: number; + arrayBuffers: number; +} + +export interface CaseValidation { + passed: boolean; + errors: string[]; +} + +export interface SuccessfulWorkerResult { + status: 'ok'; + pid: number; + caseName: string; + buildMs: number; + schemaHash: string; + schemaTypeCount: number; + runtimeVerified: true; + caseValidation: CaseValidation; + metadata?: Record; + memory: { + baseline: MemorySnapshot; + afterBuild: MemorySnapshot; + delta: MemorySnapshot; + processPeakRss: number; + }; +} + +export interface FailedWorkerResult { + status: 'error'; + pid: number; + caseName: string; + error: string; +} + +export type WorkerResult = SuccessfulWorkerResult | FailedWorkerResult; + +export interface BenchmarkRun extends BenchmarkCoordinate { + result: WorkerResult; +} + +export interface MetricSummary { + median: number; + min: number; + max: number; + samples: number[]; +} + +export interface CaseSummary { + sampleCount: number; + buildMs: MetricSummary; + heapUsedAfterBuild: MetricSummary; + heapUsedDelta: MetricSummary; + rssAfterBuild: MetricSummary; + rssDelta: MetricSummary; + processPeakRss: MetricSummary; +} + +export interface MetricComparison { + baseline: number; + candidate: number; + difference: number; + percentChange: number | null; +} + +export interface CaseComparison { + baselineCase: string; + candidateCase: string; + buildMs: MetricComparison; + heapUsedAfterBuild: MetricComparison; + heapUsedDelta: MetricComparison; + rssAfterBuild: MetricComparison; + rssDelta: MetricComparison; + processPeakRss: MetricComparison; +} + +export interface BenchmarkReport { + format: 'constructive-performance-suite/v1'; + generatedAt: string; + node: string; + platform: string; + architecture: string; + suite: BenchmarkSuiteDefinition; + config: { + repetitions: number; + seed: number; + order: string[] | null; + }; + schedule: BenchmarkCoordinate[]; + runs: BenchmarkRun[]; + validation: { + allRunsSucceeded: boolean; + freshProcessPerRun: boolean; + caseValidationPassed: boolean; + schemaGroupsEquivalent: boolean; + schemaGroups: Record; + errors: string[]; + }; + summaries: Record; +} diff --git a/packages/perf-harness/tsconfig.esm.json b/packages/perf-harness/tsconfig.esm.json new file mode 100644 index 000000000..1a3c9914f --- /dev/null +++ b/packages/perf-harness/tsconfig.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist/esm", + "module": "esnext", + "moduleResolution": "bundler" + } +} diff --git a/packages/perf-harness/tsconfig.json b/packages/perf-harness/tsconfig.json new file mode 100644 index 000000000..319daa4b0 --- /dev/null +++ b/packages/perf-harness/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "moduleResolution": "nodenext", + "module": "nodenext", + "isolatedModules": true + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce55a54f7..9ba097208 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2615,6 +2615,37 @@ importers: version: 0.3.0 publishDirectory: dist + packages/perf-harness: + dependencies: + graphile-build: + specifier: 5.1.1 + version: 5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0) + graphile-build-pg: + specifier: 5.1.3 + version: 5.1.3(@dataplan/pg@1.1.1(@dataplan/json@1.0.1(grafast@1.1.2(graphql@16.13.0)))(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.1.2(graphql@16.13.0))(graphile-build@5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-config: + specifier: 1.1.0 + version: 1.1.0 + graphql: + specifier: 16.13.0 + version: 16.13.0 + pg: + specifier: ^8.21.0 + version: 8.21.0 + postgraphile: + specifier: 5.1.4 + version: 5.1.4(f282a162d8bd20a217e08c60f5396af8) + devDependencies: + '@types/node': + specifier: ^22.19.11 + version: 22.19.19 + '@types/pg': + specifier: ^8.20.4 + version: 8.20.4 + makage: + specifier: ^0.3.0 + version: 0.3.0 + packages/postmaster: dependencies: 12factor-env: