Add a reusable Graphile performance harness

packages/perf-harness/README.md

@@ -0,0 +1,39 @@
+# Graphile performance harness
+
+Reusable infrastructure for measuring Graphile schema builds in fresh Node
+processes. The core accepts any list of serializable benchmark cases; it does not
+interpret case names or optimization-specific configuration.
+
+Each measurement receives a new PID, starts Node with `--expose-gc`, runs a
+deterministic GC sequence, records build time and memory metrics, validates a
+runtime query, and reports a schema hash. Cases can opt into schema equivalence
+groups and provide their own lifecycle validation through the worker result.
+
+## Extending the harness
+
+Define a suite and provide a dedicated worker entry point:
+
+```ts
+const suite = {
+  name: 'example',
+  cases: [
+    {
+      name: 'baseline',
+      workerConfig: { schemas: ['cperf_example'] },
+      expectedSchemaGroup: 'example-schema',
+    },
+  ],
+};
+
+await runBenchmarkSuite(suite, options, workerPath);
+```
+
+`workerConfig` must be JSON-serializable. Logic is implemented in the worker
+entry rather than serializing functions across process boundaries.
+
+The package includes `stock-worker.js` as a minimal upstream Graphile baseline.
+Database credentials are supplied through `CPERF_DATABASE_URL` and are redacted
+from worker failures and JSON reports.
+
+The PostgreSQL fixture command only creates a previously absent schema whose
+name starts with `cperf_`; it never drops or replaces schemas.

packages/perf-harness/__tests__/fixture.test.ts

@@ -0,0 +1,22 @@
+import {
+  validateFixtureSchema,
+  validateFixtureTableCount,
+} from '../src/fixture';
+
+describe('fixture safety', () => {
+  test('only accepts narrowly scoped benchmark schema names', () => {
+    expect(validateFixtureSchema('cperf_example_1')).toBe('cperf_example_1');
+    expect(() => validateFixtureSchema('public')).toThrow(
+      'must start with cperf_'
+    );
+    expect(() =>
+      validateFixtureSchema('cperf_example; drop schema public')
+    ).toThrow('must start with cperf_');
+  });
+
+  test('bounds generated fixture size', () => {
+    expect(validateFixtureTableCount(64)).toBe(64);
+    expect(() => validateFixtureTableCount(0)).toThrow('between 1 and 500');
+    expect(() => validateFixtureTableCount(501)).toThrow('between 1 and 500');
+  });
+});

packages/perf-harness/__tests__/fixtures/fake-worker.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const envelope = JSON.parse(
+  Buffer.from(process.env.CPERF_WORKER_CONFIG, 'base64url').toString('utf8')
+);
+const value = envelope.workerConfig.value;
+const memory = {
+  rss: value,
+  heapTotal: value,
+  heapUsed: value,
+  external: value,
+  arrayBuffers: value,
+};
+const result = {
+  status: 'ok',
+  pid: process.pid,
+  caseName: envelope.caseName,
+  buildMs: value,
+  schemaHash: envelope.workerConfig.schemaHash,
+  schemaTypeCount: 10,
+  runtimeVerified: true,
+  caseValidation: { passed: true, errors: [] },
+  memory: {
+    baseline: memory,
+    afterBuild: memory,
+    delta: memory,
+    processPeakRss: value,
+  },
+};
+process.stdout.write(`CPERF_RESULT ${JSON.stringify(result)}\n`);

packages/perf-harness/__tests__/process.test.ts

@@ -0,0 +1,22 @@
+import { resolve } from 'node:path';
+
+import { runWorkerProcess } from '../src/process';
+
+describe('fresh worker process', () => {
+  test('uses distinct PIDs and does not expose the database URL', async () => {
+    const worker = resolve(__dirname, 'fixtures/fake-worker.js');
+    const definition = {
+      name: 'baseline',
+      workerConfig: { value: 1, schemaHash: 'same' },
+    };
+    const databaseUrl = 'postgres://secret@example.test/database';
+    const first = await runWorkerProcess(worker, databaseUrl, definition);
+    const second = await runWorkerProcess(worker, databaseUrl, definition);
+    expect(first.pid).not.toBe(process.pid);
+    expect(second.pid).not.toBe(process.pid);
+    expect(first.pid).not.toBe(second.pid);
+    expect(JSON.stringify([first.result, second.result])).not.toContain(
+      databaseUrl
+    );
+  });
+});

packages/perf-harness/__tests__/report.test.ts

@@ -0,0 +1,78 @@
+import {
+  compareCases,
+  summarizeCase,
+  validateSchemaGroups,
+} from '../src/report';
+import type { BenchmarkRun, SuccessfulWorkerResult } from '../src/types';
+
+const result = (caseName: string, value: number): SuccessfulWorkerResult => ({
+  status: 'ok',
+  pid: value,
+  caseName,
+  buildMs: value,
+  schemaHash: 'same',
+  schemaTypeCount: 10,
+  runtimeVerified: true,
+  caseValidation: { passed: true, errors: [] },
+  memory: {
+    baseline: {
+      rss: 10,
+      heapTotal: 10,
+      heapUsed: 10,
+      external: 10,
+      arrayBuffers: 10,
+    },
+    afterBuild: {
+      rss: value,
+      heapTotal: value,
+      heapUsed: value,
+      external: value,
+      arrayBuffers: value,
+    },
+    delta: {
+      rss: value - 10,
+      heapTotal: value - 10,
+      heapUsed: value - 10,
+      external: value - 10,
+      arrayBuffers: value - 10,
+    },
+    processPeakRss: value,
+  },
+});
+
+describe('generic reports', () => {
+  test('summarizes arbitrary cases and compares medians', () => {
+    const runs: BenchmarkRun[] = [10, 30, 20].map((value, index) => ({
+      repetition: index + 1,
+      position: 1,
+      caseName: 'base',
+      result: result('base', value),
+    }));
+    runs.push(
+      ...[5, 15, 10].map((value, index) => ({
+        repetition: index + 1,
+        position: 2,
+        caseName: 'candidate',
+        result: result('candidate', value),
+      }))
+    );
+    const base = summarizeCase(runs, 'base')!;
+    const candidate = summarizeCase(runs, 'candidate')!;
+    expect(
+      compareCases('base', 'candidate', base, candidate).buildMs.percentChange
+    ).toBe(-50);
+    expect(
+      validateSchemaGroups(
+        [
+          { name: 'base', workerConfig: null, expectedSchemaGroup: 'schema' },
+          {
+            name: 'candidate',
+            workerConfig: null,
+            expectedSchemaGroup: 'schema',
+          },
+        ],
+        runs
+      )
+    ).toEqual({ equivalent: true, hashes: { schema: 'same' }, errors: [] });
+  });
+});

packages/perf-harness/__tests__/run.test.ts

@@ -0,0 +1,37 @@
+import { resolve } from 'node:path';
+
+import { runBenchmarkSuite } from '../src/run';
+
+describe('generic suite runner', () => {
+  test('validates fresh processes and schema groups without fixed case names', async () => {
+    const report = await runBenchmarkSuite(
+      {
+        name: 'test-suite',
+        cases: ['alpha', 'beta', 'gamma'].map((name, index) => ({
+          name,
+          workerConfig: { value: index + 1, schemaHash: 'same' },
+          expectedSchemaGroup: 'schema',
+        })),
+      },
+      {
+        databaseUrl: 'postgres:///not-used-by-fake-worker',
+        repetitions: 1,
+        seed: 1,
+        order: ['alpha', 'beta', 'gamma'],
+      },
+      resolve(__dirname, 'fixtures/fake-worker.js')
+    );
+    expect(report.validation).toEqual(
+      expect.objectContaining({
+        allRunsSucceeded: true,
+        freshProcessPerRun: true,
+        caseValidationPassed: true,
+        schemaGroupsEquivalent: true,
+        schemaGroups: { schema: 'same' },
+        errors: [],
+      })
+    );
+    expect(new Set(report.runs.map((run) => run.result.pid)).size).toBe(3);
+    expect(JSON.stringify(report)).not.toContain('postgres:///');
+  });
+});

packages/perf-harness/__tests__/schedule.test.ts

@@ -0,0 +1,29 @@
+import { makeSchedule } from '../src/schedule';
+
+const cases = [
+  { name: 'a', workerConfig: null },
+  { name: 'b', workerConfig: null },
+  { name: 'c', workerConfig: null },
+];
+
+describe('generic benchmark scheduling', () => {
+  test('is deterministic and supports any case list', () => {
+    const first = makeSchedule(cases, 4, 1234);
+    expect(makeSchedule(cases, 4, 1234)).toEqual(first);
+    expect(makeSchedule(cases, 4, 4321)).not.toEqual(first);
+    for (let repetition = 1; repetition <= 4; repetition += 1) {
+      expect(
+        first
+          .filter((item) => item.repetition === repetition)
+          .map((item) => item.caseName)
+          .sort()
+      ).toEqual(['a', 'b', 'c']);
+    }
+  });
+
+  test('accepts an exact order containing each case once', () => {
+    expect(
+      makeSchedule(cases, 2, 1, ['c', 'a', 'b']).map((item) => item.caseName)
+    ).toEqual(['c', 'a', 'b', 'c', 'a', 'b']);
+  });
+});

packages/perf-harness/jest.config.js

@@ -0,0 +1,12 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
+  },
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*'],
+  testPathIgnorePatterns: ['/__tests__/fixtures/'],
+};

packages/perf-harness/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@constructive-io/perf-harness",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Reusable fresh-process Graphile performance harness",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "bin": {
+    "cperf": "index.js"
+  },
+  "scripts": {
+    "clean": "makage clean",
+    "build": "makage build",
+    "build:dev": "makage build --dev",
+    "lint": "eslint . --fix",
+    "test": "jest"
+  },
+  "dependencies": {
+    "graphile-build": "5.1.1",
+    "graphile-build-pg": "5.1.3",
+    "graphile-config": "1.1.0",
+    "graphql": "16.13.0",
+    "pg": "^8.21.0",
+    "postgraphile": "5.1.4"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.11",
+    "@types/pg": "^8.20.4",
+    "makage": "^0.3.0"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "license": "MIT"
+}