diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6930daea56..e5dc0160c37 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -470,7 +470,7 @@ importers: dependencies: '@convex-dev/sharded-counter': specifier: ^0.2.0 - version: 0.2.0(convex@1.32.0(react@19.2.4)) + version: 0.2.0(convex@1.38.0(react@19.2.4)) '@supabase/supabase-js': specifier: ^2.80.0 version: 2.97.0 @@ -521,8 +521,8 @@ importers: specifier: ^1.3.2 version: 1.3.9 convex: - specifier: ^1.29.0 - version: 1.32.0(react@19.2.4) + specifier: ^1.37.0 + version: 1.38.0(react@19.2.4) dotenv: specifier: ^17.2.3 version: 17.2.3 @@ -546,7 +546,7 @@ importers: dependencies: nuxt: specifier: ~3.16.0 - version: 3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2) + version: 3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2) spacetimedb: specifier: workspace:* version: link:../../crates/bindings-typescript @@ -7311,7 +7311,6 @@ packages: bun@1.3.9: resolution: {integrity: sha512-v5hkh1us7sMNjfimWE70flYbD5I1/qWQaqmJ45q2qk5H/7muQVa478LSVRSFyGTBUBog2LsPQnfIRdjyWJRY+A==} - cpu: [arm64, x64] os: [darwin, linux, win32] hasBin: true @@ -7705,19 +7704,22 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - convex@1.32.0: - resolution: {integrity: sha512-5FlajdLpW75pdLS+/CgGH5H6yeRuA+ru50AKJEYbJpmyILUS+7fdTvsdTaQ7ZFXMv0gE8mX4S+S3AtJ94k0mfw==} + convex@1.38.0: + resolution: {integrity: sha512-122AC6y5lUS7mr39cluLw9+TOtRX5d/XxeivHhHObs/NTXoVvOnIgDzexVcxaz6Rk0oLFSoydSR1rDCltEz/0A==} engines: {node: '>=18.0.0', npm: '>=7.0.0'} hasBin: true peerDependencies: '@auth0/auth0-react': ^2.0.1 '@clerk/clerk-react': ^4.12.8 || ^5.0.0 + '@clerk/react': ^6.4.3 react: ^18.0.0 || ^19.0.0-0 || ^19.0.0 peerDependenciesMeta: '@auth0/auth0-react': optional: true '@clerk/clerk-react': optional: true + '@clerk/react': + optional: true react: optional: true @@ -16608,9 +16610,9 @@ snapshots: '@colors/colors@1.5.0': optional: true - '@convex-dev/sharded-counter@0.2.0(convex@1.32.0(react@19.2.4))': + '@convex-dev/sharded-counter@0.2.0(convex@1.38.0(react@19.2.4))': dependencies: - convex: 1.32.0(react@19.2.4) + convex: 1.38.0(react@19.2.4) '@cspotcode/source-map-support@0.8.1': dependencies: @@ -19433,11 +19435,11 @@ snapshots: transitivePeerDependencies: - supports-color - '@nuxt/cli@3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.3.5)': + '@nuxt/cli@3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.5.1)': dependencies: '@bomb.sh/tab': 0.0.12(cac@6.7.14)(citty@0.2.0) '@clack/prompts': 1.0.0 - c12: 3.3.3(magicast@0.3.5) + c12: 3.3.3(magicast@0.5.1) citty: 0.2.0 confbox: 0.2.4 consola: 3.4.2 @@ -19533,9 +19535,9 @@ snapshots: - utf-8-validate - vue - '@nuxt/kit@3.16.2(magicast@0.3.5)': + '@nuxt/kit@3.16.2(magicast@0.5.1)': dependencies: - c12: 3.3.3(magicast@0.3.5) + c12: 3.3.3(magicast@0.5.1) consola: 3.4.2 defu: 6.1.4 destr: 2.0.5 @@ -19593,18 +19595,18 @@ snapshots: pathe: 2.0.3 std-env: 3.10.0 - '@nuxt/telemetry@2.7.0(@nuxt/kit@3.16.2(magicast@0.3.5))': + '@nuxt/telemetry@2.7.0(@nuxt/kit@3.16.2(magicast@0.5.1))': dependencies: - '@nuxt/kit': 3.16.2(magicast@0.3.5) + '@nuxt/kit': 3.16.2(magicast@0.5.1) citty: 0.2.0 consola: 3.4.2 ofetch: 2.0.0-alpha.3 rc9: 3.0.0 std-env: 3.10.0 - '@nuxt/vite-builder@3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)': + '@nuxt/vite-builder@3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2)': dependencies: - '@nuxt/kit': 3.16.2(magicast@0.3.5) + '@nuxt/kit': 3.16.2(magicast@0.5.1) '@rollup/plugin-replace': 6.0.3(rollup@4.56.0) '@vitejs/plugin-vue': 5.2.4(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3)) '@vitejs/plugin-vue-jsx': 4.2.0(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3)) @@ -24408,7 +24410,7 @@ snapshots: convert-source-map@2.0.0: {} - convex@1.32.0(react@19.2.4): + convex@1.38.0(react@19.2.4): dependencies: esbuild: 0.27.0 prettier: 3.6.2 @@ -28425,19 +28427,19 @@ snapshots: schema-utils: 3.3.0 webpack: 5.102.0 - nuxt@3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2): + nuxt@3.16.2(@parcel/watcher@2.5.6)(@types/node@24.3.0)(better-sqlite3@12.6.2)(cac@6.7.14)(db0@0.3.4(better-sqlite3@12.6.2)(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0)))(drizzle-orm@0.44.7(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.16.0)(@types/sql.js@1.4.9)(better-sqlite3@12.6.2)(bun-types@1.3.11)(pg@8.18.0)(sql.js@1.14.0))(encoding@0.1.13)(eslint@9.33.0(jiti@2.6.1))(ioredis@5.9.2)(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue-tsc@2.2.12(typescript@5.6.3))(yaml@2.8.2): dependencies: - '@nuxt/cli': 3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.3.5) + '@nuxt/cli': 3.33.1(@nuxt/schema@3.16.2)(cac@6.7.14)(magicast@0.5.1) '@nuxt/devalue': 2.0.2 '@nuxt/devtools': 2.7.0(vite@6.4.1(@types/node@24.3.0)(jiti@2.6.1)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(yaml@2.8.2))(vue@3.5.26(typescript@5.6.3)) - '@nuxt/kit': 3.16.2(magicast@0.3.5) + '@nuxt/kit': 3.16.2(magicast@0.5.1) '@nuxt/schema': 3.16.2 - '@nuxt/telemetry': 2.7.0(@nuxt/kit@3.16.2(magicast@0.3.5)) - '@nuxt/vite-builder': 3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.3.5)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2) + '@nuxt/telemetry': 2.7.0(@nuxt/kit@3.16.2(magicast@0.5.1)) + '@nuxt/vite-builder': 3.16.2(@types/node@24.3.0)(eslint@9.33.0(jiti@2.6.1))(magicast@0.5.1)(optionator@0.9.4)(rollup@4.56.0)(sass@1.97.1)(terser@5.43.1)(tsx@4.21.0)(typescript@5.6.3)(vue-tsc@2.2.12(typescript@5.6.3))(vue@3.5.26(typescript@5.6.3))(yaml@2.8.2) '@oxc-parser/wasm': 0.60.0 '@unhead/vue': 2.1.4(vue@3.5.26(typescript@5.6.3)) '@vue/shared': 3.5.26 - c12: 3.3.3(magicast@0.3.5) + c12: 3.3.3(magicast@0.5.1) chokidar: 4.0.3 compatx: 0.1.8 consola: 3.4.2 @@ -32575,7 +32577,6 @@ snapshots: ws@8.18.3: {} - wsl-utils@0.1.0: dependencies: is-wsl: 3.1.0 diff --git a/templates/keynote-2/.env.example b/templates/keynote-2/.env.example index 2811d42abd2..6515bc0bcb4 100644 --- a/templates/keynote-2/.env.example +++ b/templates/keynote-2/.env.example @@ -57,7 +57,19 @@ SUPABASE_RPC_URL=http://127.0.0.1:4106 # ===== Seeding knobs ===== SEED_ACCOUNTS=100000 -SEED_INITIAL_BALANCE=10000000 +SEED_INITIAL_BALANCE=1000000000 + +# ===== Bench knobs ===== +# Pool size for pg-based RPC servers (postgres, cockroach, supabase, planetscale). Default: 64. +# Read at RPC-server startup — restart the RPC if you change this. +MAX_POOL=64 + +# Pipelining for the bench client. Bench pipelining is global across connectors. +# Some connectors may still have their own internal transport details. +# Setting MAX_INFLIGHT_PER_WORKER alone does NOT enable pipelining for them. +# If BENCH_PIPELINED=1, you must set MAX_INFLIGHT_PER_WORKER explicitly. +#BENCH_PIPELINED=1 +#MAX_INFLIGHT_PER_WORKER=40 VERIFY=0 ENABLE_RPC_SERVERS=0 diff --git a/templates/keynote-2/DEVELOP.md b/templates/keynote-2/DEVELOP.md index d99c1ffedb4..11488a55bb8 100644 --- a/templates/keynote-2/DEVELOP.md +++ b/templates/keynote-2/DEVELOP.md @@ -27,8 +27,8 @@ The script will: **Options:** -- `--seconds N` - Benchmark duration (default: 10) -- `--concurrency N` - Concurrent connections (default: 50) +- `--seconds N` - Benchmark duration (default: 300) +- `--concurrency N` - Concurrent connections (default: 64) - `--alpha N` - Contention level (default: 1.5) - `--systems a,b,c` - Systems to compare (default: convex,spacetimedb) - `--stdb-compression none|gzip` - SpacetimeDB client compression mode (default: none) @@ -193,26 +193,33 @@ pnpm run bench test-1 --connectors spacetimedb --stdb-compression gzip # Only run selected connectors pnpm run bench test-1 --connectors spacetimedb,sqlite_rpc + +# Sweep alpha values for a connector set +pnpm run bench test-1 --alpha 0,1.5 --connectors postgres_rpc,bun --seconds 300 + +# Sweep contention (alpha) for a single connector: start,end,step,concurrency +pnpm run bench test-1 --connectors cockroach_rpc --contention-tests 0,1.5,0.5,64 + +# Sweep concurrency for a single connector: start,end,factor,alpha +pnpm run bench test-1 --connectors cockroach_rpc --concurrency-tests 16,512,2,1.5 ``` ## CLI Arguments -From `src/cli.ts`: - - **`test-name`** (positional) - Name of the test folder under `src/tests/` - Default: `test-1` - **`--seconds N`** - Duration of the benchmark in seconds - - Default: `10` + - Default: `300` - **`--concurrency N`** - Number of workers / in-flight operations - - Default: `50` + - Default: `64` - **`--alpha A`** - - Zipf α parameter for account selection (hot vs cold distribution) + - Zipf alpha parameter for account selection (hot vs cold distribution) - Default: `1.5` - **`--connectors list`** @@ -227,15 +234,34 @@ From `src/cli.ts`: - The valid names come from `tc.system` in the test modules and the keys in `CONNECTORS` - Valid names: `convex`, `spacetimedb`, `bun`, `postgres_rpc`, `cockroach_rpc`, `sqlite_rpc`, `supabase_rpc`, `planetscale_pg_rpc` -- **`--contention-tests startAlpha endAlpha step concurrency`** - - Runs a sweep over Zipf α values for a single connector - - Uses `startAlpha`, `endAlpha`, and `step` to choose the α values - - Uses the provided `concurrency` for all runs +- **`--systems list`** + - Alias for `--connectors` in bench mode + +- **`--runs N`** + - Repeat each `(connector, alpha)` combination `N` times + - Default: `1` + +- **`--prep-between-alphas`** + - Run `pnpm run prep` before each `(connector, alpha)` combination + +- **`--contention-tests start,end,step,concurrency`** + - Sweep Zipf alpha values for one connector -- **`--concurrency-tests startConc endConc step alpha`** - - Runs a sweep over concurrency levels for a single connector - - Uses `startConc`, `endConc`, and `step` to choose the concurrency values - - Uses the provided `alpha` for all runs +- **`--concurrency-tests start,end,factor,alpha`** + - Sweep concurrency values for one connector + +- **`--bench-pipelined` / `--no-bench-pipelined`** + - Force pipelining on or off across connectors + +- **`--max-inflight-per-worker N`** + - Max in-flight requests per worker when pipelining is enabled + - Required when `--bench-pipelined` is enabled + +- **`--log-errors`** + - Log per-operation errors during runs + +- **`--verify-transactions`** + - Run connector verification at end of run --- @@ -244,7 +270,7 @@ From `src/cli.ts`: You can also run the benchmark via Docker instead of Node directly: ```bash -docker compose run --rm bench -- --seconds 5 --concurrency 50 --alpha 1 --connectors convex +docker compose run --rm bench -- --seconds 5 --concurrency 64 --alpha 1 --connectors convex ``` If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `pnpm run prep` before running bench. @@ -254,7 +280,10 @@ If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compos Every run writes a JSON file into `./runs/`: - Directory: `./runs/` -- Filename: `-.json` - - Example: `test-1-2025-11-17T16-45-12-345Z.json` +- Filename: `--a-.json` + - Example: `test-1-postgres_rpc-a1.5-2025-11-17T16-45-12-345Z.json` + +For rollup tables, compute steady-state stats after a 30-second warmup window (`tSec >= 30`). The `scripts/bench-stats.py` default matches this (`--warmup-sec 30`). + +Point your visualizations / CSV exports at `./runs/` and you're good. -Point your visualizations / CSV exports at `./runs/` and you’re good. diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 9f3cbc9e70f..05313556944 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -20,39 +20,84 @@ The demo compares SpacetimeDB and Convex by default, since both are easy for any ## Results Summary -All tests use 50 concurrent connections with a transfer workload (read-modify-write transaction between two accounts). +For all tests, we ran N clients where N is 2x the number of CPUs on the database machine used for the test. Exact client counts are shown in each row. The workload is a transfer transaction (read-modify-write transaction between two accounts). + +The SpacetimeDB rows were obtained using a single-node SpacetimeDB Standalone instance, so the published numbers are reproducible with the public, downloadable server. + +Each row reports mean TPS and sample standard deviation of per-second throughput within a single 300-second run. `alpha=1.5` corresponds to ~80% contention. When standard deviation approaches or exceeds mean TPS, throughput is unstable across the run. + +Data description: reported summary metrics are computed from steady-state windows after a 30-second warmup (`tSec >= 30`), using the recorded per-second `timeSeries` data. + +### Alpha = 0 + +| System | clients | pipelining | max_pool | TPS | TPS Stddev | p50 lat ms | p99 lat ms | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| SpacetimeDB | 64 | 40 | N/A | 279,024 | 4,763 | 8 | 12 | +| Node.js + SQLite | 64 | off | N/A | 3,121 | 80 | 19 | 40 | +| Node.js + Supabase | 64 | off | 64 | 7,362 | 1,179 | 6 | 18 | +| Bun + Postgres | 64 | off | 64 | 10,729 | 146 | 5 | 11 | +| Node.js + Postgres | 64 | off | 64 | 9,904 | 223 | 6 | 11 | +| Node.js + PlanetScale (SN) | 64 | off | 64 | 4,535 | 117 | 14 | 20 | +| Node.js + PlanetScale (HA) | 384 | off | 384 | 4,275 | 135 | 89 | 110 | +| Convex | 64 | off | N/A | 1,140 | 118 | 53 | 62 | +| Node.js + CockroachDB (5 node) | 320 | off | 320 | 4,253 | 561 | 71 | 120 | +| HAProxy - Node.js + CockroachDB (5 node) | 320 | off | 320 | 5,481 | 566 | 57 | 95 | + +### Alpha = 1.5 + +| System | clients | pipelining | max_pool | TPS | TPS Stddev | p50 lat ms | p99 lat ms | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| SpacetimeDB | 64 | 40 | N/A | 303,919 | 4,712 | 7 | 11 | +| Node.js + SQLite | 64 | off | N/A | 3,188 | 73 | 18 | 39 | +| Node.js + Supabase | 64 | off | 64 | 2,534 | 57 | 2 | 197 | +| Bun + Postgres | 64 | off | 64 | 2,772 | 61 | 7 | 13 | +| Node.js + Postgres | 64 | off | 64 | 961 | 25 | 10 | 16 | +| Node.js + PlanetScale (SN) | 64 | off | 64 | 235 | 12 | 20 | 2,504 | +| Node.js + PlanetScale (HA) | 384 | off | 384 | 248 | 13 | 416 | 10,121 | +| Convex | 64 | off | N/A | 126 | 52 | 20 | 1,081 | +| Node.js + CockroachDB (5 node) | 320 | off | 320 | 0.03 | 0.18 | 698 | 9,695 | +| HAProxy - Node.js + CockroachDB (5 node) | 64 | off | 64 | 6.87 | 9.12 | 5,943 | 9,880 | + +Note: the HAProxy + CockroachDB `alpha=1.5` row uses 64 clients (instead of 320) because 320-way concurrency overwhelmed CRDB and did not produce stable sample data for this profile. + +### Alpha = 0 (All-Connectors Pipelining Check) + +The headline comparison allows pipelining only for SpacetimeDB. This separate check enables pipelining for every connector to show how the other systems behave when clients submit up to 40 requests without waiting for each response. + +| System | clients | pipelining | max_pool | TPS | TPS Stddev | p50 lat ms | p99 lat ms | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Node.js + SQLite | 64 | 40 | N/A | 2,977 | 84 | 722 | 747 | +| Node.js + Supabase | 64 | 40 | 64 | 8,874 | 308 | 284 | 303 | +| Bun + Postgres | 64 | 40 | 64 | 10,184 | 120 | 250.1 | 260.5 | +| Node.js + Postgres | 64 | 40 | 64 | 9,165 | 145 | 276 | 290 | +| Node.js + PlanetScale (SN) | 64 | 40 | 64 | 4,325 | 85 | 590 | 604 | +| Node.js + PlanetScale (HA) | 384 | 40 | 384 | 3,355 | 327 | 4,354 | 4,438 | +| Convex | 64 | 40 | N/A | 1,154 | 134 | 2,119 | 2,150 | +| Node.js + CockroachDB (5 node) | 320 | 40 | 320 | 4,250 | 766 | 3,030 | 3,161 | +| HAProxy - Node.js + CockroachDB (5 node) | 320 | 40 | 320 | 5,992 | 1,765 | 2,431 | 2,562 | + +**Key Finding:** In these runs, SpacetimeDB is the only system sustaining hundreds of thousands of TPS in both alpha profiles. At `alpha=0`, the strongest non-SpacetimeDB results are in the ~10k TPS range, while at `alpha=1.5` several systems show severe contention sensitivity with large tail-latency growth and throughput collapse. -| System | TPS (~0% Contention) | TPS (~80% Contention) | -| --------------------------------- | -------------------- | --------------------- | -| SpacetimeDB (TypeScript Module) | | 307,074 | -| SpacetimeDB (Rust Module) | | 265,542 | -| SQLite + Node HTTP + Drizzle | | 3,236 | -| Bun + Drizzle + Postgres | 7,115 | 2,074 | -| Postgres + Node HTTP + Drizzle | 6,429 | 2,798 | -| Supabase + Node HTTP + Drizzle | 6,310 | 1,268 | -| CockroachDB + Node HTTP + Drizzle | 5,129 | 197 | -| PlanetScale + Node HTTP + Drizzle | 477 | 30 | -| Convex | 438 | 58 | - -**Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite RPC at 3,236 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). - -### Contention Impact +## Methodology -![Contention Chart](./contention-chart.png) +All systems were tested with **out-of-the-box database and platform settings**, with one exception: the local Postgres instance (and Bun, which uses the same Postgres instance) is configured with `default_transaction_isolation = 'serializable'`. For Postgres-like RPC servers, the app-side Drizzle connection pool is configured as shown in the result tables, and the benchmark connects directly to Postgres. -The chart above shows TPS vs Zipf Alpha (contention level). Higher alpha values concentrate more transactions on fewer "hot" accounts, increasing contention. SpacetimeDB maintains consistent performance regardless of contention level, while traditional database architectures show significant degradation. +The managed Postgres services (Supabase, PlanetScale) run at their default isolation level of `READ COMMITTED`. -## Methodology +Throughput is counted from successful operations that the benchmark client observes completing inside the configured test window for every system. -All systems were tested with **out-of-the-box default settings** - no custom tuning, no configuration optimization. This reflects what developers experience when they first adopt these technologies. +### Published Benchmark Defaults -For cloud services, we tested paid tiers to give them their best chance: +The reported tables in this README use the following profile defaults unless a row explicitly shows a different value: -- **PlanetScale**: PS-2560 (32 vCPUs, 256 GB RAM), single node, us-central1. -- **Supabase**: Pro tier -- **Convex**: Pro tier +- `clients`: N clients where N is 2x the number of CPUs on the database machine used for the test +- `pipelining`: `off` for non-pipelined runs +- `MAX_POOL`: `64` for pg-based RPC servers (`postgres_rpc`, `cockroach_rpc`, `supabase_rpc`, `planetscale_pg_rpc`) +- Main comparison runs use `MAX_INFLIGHT_PER_WORKER=40` for SpacetimeDB only +- All-connectors pipelining-check runs use `BENCH_PIPELINED=1` and `MAX_INFLIGHT_PER_WORKER=40` +- When `BENCH_PIPELINED=1`, set `MAX_INFLIGHT_PER_WORKER` explicitly in the environment -The reported SpacetimeDB module results were run against a 5-way replicated cluster rather than a single standalone node. +For rows that scale client count above 64 (for example, some HA topologies), `max_pool` is scaled to match the row values shown in the table. ### Test Architecture @@ -70,7 +115,15 @@ Client → Integrated Platform (compute + storage colocated) This ensures we're measuring real-world application performance, not raw database throughput. -Throughput is counted from successful operations that the benchmark client observes completing inside the configured test window for every system. +### Machine Topology + +The reported numbers use a single benchmark host wherever possible. This means client, server, and database were all run on the same machine. + +We did this mainly because it was the most favorable benchmarking setup for the competitor platforms, because it minimizes server to database latency, but also because it allows others to easily reproduce the results. + +For completeness, we also tested separated-machine topologies, where the benchmark client, server, and database processes were not colocated on one machine. However, in each case we found that doing so either did not change or reduced the throughput of other systems due to the additional network hop. We published the most favorable numbers for our competitors. + +The platforms that cannot use this exact topology are PlanetScale and CockroachDB. PlanetScale operates a managed cloud database and does not have a self-hosted variant of the service, so the benchmark client and RPC server are colocated on a benchmark host in the same region and availability zone as the database host. CockroachDB is a distributed database running across multiple nodes, so the benchmark client and RPC server cannot be colocated with the database on a single node. ### The Transaction @@ -86,48 +139,42 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command +The numbers in the table above were collected with `pnpm run bench`: + ```bash -docker compose run --rm bench -- --seconds 10 --concurrency 50 --alpha XX --connectors YY +pnpm install +pnpm run prep # seed all backing databases once +pnpm run bench --alpha 0,1.5 --connectors --seconds 300 # one JSON per (connector, alpha) ``` -- `--seconds 10`: Duration of benchmark run -- `--concurrency 50`: Number of concurrent client connections -- `--alpha 0`: ~0% contention (uniform account distribution) -- `--alpha 1.5`: ~80% contention (Zipf distribution concentrating on hot accounts) -- `--stdb-compression none|gzip`: SpacetimeDB client compression mode (default: `none`) - -### Hardware Configuration +`--alpha` and `--connectors` both accept comma-separated values. The bench writes one JSON per (connector, alpha, run) tuple into `runs/`. -**Server Machine (Variant A - PhoenixNAP):** +When aggregating these JSONs into summary tables, use a 30-second warmup cutoff (`--warmup-sec 30`) to match the published numbers. -- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 +Useful flags: -**Server Machine (Variant B - Google Cloud):** +- `--alpha `: Zipf alpha. This benchmark reports `0` (uniform / ~0% contention) and `1.5` (Zipf / ~80% contention). +- `--connectors `: which connectors to run. Defaults to every test in `src/tests/test-1/`. +- `--seconds `: duration of each run. +- `--concurrency `: number of concurrent clients (default: `64`). +- `--runs `: repeat each (connector, alpha) combination this many times (default: `1`). Each repeat writes its own JSON. +- `--prep-between-alphas`: run `pnpm run prep` before each (connector, alpha) combination to reset DB state. +- `--stdb-compression `: SpacetimeDB client compression mode (default: `none`). -- c4-standard-32-lssd (32 vCPUs, 120 GB Memory) OS: Ubuntu 24.04 -- RAID 0 on 5 Local SSDs -- Region: us-central1 - -**Client Machine:** - -- c4-standard-32 (32 vCPUs, 120 GB Memory) OS: Ubuntu 24.04 -- Region: us-central1 -- Runs on a **separate machine** from the server +### Hardware Configuration -**Note:** All services (databases, web servers, benchmark runner) except Convex local dev backend run in the same Docker environment on the server machine. +**Server Machine (all systems except PlanetScale):** -### Why Separate Client Machines? +- PhoenixNAP s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 -Running clients on separate machines ensures: +**Bench client for PlanetScale:** -- Network round-trip latency is measured (realistic production scenario) -- Client CPU/memory doesn't compete with server resources -- Results reflect actual deployment conditions +- AWS `m7i.8xlarge` in `us-east-2`, colocated with the PlanetScale cluster. Clusters tested: PS-2560 single-node EBS, M-15360 Metal HA (1 primary + 2 replicas). Both Postgres 18.3. ### Account Seeding - 100,000 accounts seeded before each benchmark -- Initial balance: 10,000,000 per account +- Initial balance: 1,000,000,000 per account - Zipf distribution controls which accounts are selected for transfers ## Technical Notes @@ -152,7 +199,7 @@ This architectural difference means SpacetimeDB can execute transactions in micr ### Client Pipelining -The benchmark supports **pipelining** for all clients - sending multiple requests without waiting for responses. This maximizes throughput by keeping connections saturated. +The benchmark supports **pipelining** for all clients - sending multiple requests without waiting for responses. The headline comparison uses this for SpacetimeDB only; the all-connectors pipelining check enables it across systems. ### Confirmed Reads (`withConfirmedReads`) @@ -160,13 +207,13 @@ SpacetimeDB supports `withConfirmedReads` mode which ensures transactions are du ### Cloud vs Local Results -PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud database latency**. When the database is accessed over the network (even within the same cloud region), round-trip latency dominates performance. This is why SpacetimeDB's colocated architecture provides such dramatic improvements. +PlanetScale results (~280 TPS under high contention, regardless of cluster tier) demonstrate the **significant impact of cloud database latency**. When the database is accessed over the network (even within the same cloud region), round-trip latency dominates performance. This is why SpacetimeDB's colocated architecture provides such dramatic improvements. ## Systems Tested | System | Architecture | | --------------------------------- | ------------------------------------------------------- | -| SpacetimeDB | Integrated platform. | +| SpacetimeDB Standalone | Integrated platform; single-node downloadable server. | | SQLite + Node HTTP + Drizzle | Node.js HTTP server → Drizzle ORM → SQLite | | Bun + Drizzle + Postgres | Bun HTTP server → Drizzle ORM → PostgreSQL | | Postgres + Node HTTP + Drizzle | Node.js HTTP server → Drizzle ORM → PostgreSQL | @@ -186,3 +233,4 @@ Benchmark results are written to `./runs/` as JSON files with TPS and latency st ## License See repository root for license information. + diff --git a/templates/keynote-2/bun/bun-server.ts b/templates/keynote-2/bun/bun-server.ts index 08f769d733d..be518b625f3 100644 --- a/templates/keynote-2/bun/bun-server.ts +++ b/templates/keynote-2/bun/bun-server.ts @@ -3,15 +3,18 @@ import { drizzle } from 'drizzle-orm/node-postgres'; import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core'; import { eq, inArray, sql } from 'drizzle-orm'; import { RpcRequest, RpcResponse } from '../src/connectors/rpc/rpc_common'; -// import { poolMaxFromEnv } from '../src/helpers'; +import { getSharedRuntimeDefaults } from '../src/config.ts'; +import { withTxnRetry } from '../src/rpc-servers/retry.ts'; const DB_URL = process.env.BUN_PG_URL ?? process.env.PG_URL; if (!DB_URL) throw new Error('BUN_PG_URL or PG_URL not set'); +const { poolMax } = getSharedRuntimeDefaults(); + const pool = new Pool({ connectionString: DB_URL, application_name: 'bun-rpc-drizzle', - // max: poolMaxFromEnv() + max: poolMax, }); const accounts = pgTable('accounts', { @@ -121,37 +124,39 @@ async function rpcTransfer(args: Record) { const delta = BigInt(amount); - await db.transaction(async (tx) => { - // Lock both rows in a deterministic order to avoid deadlocks - const rows = await tx - .select() - .from(accounts) - .where(inArray(accounts.id, [fromId, toId])) - .for('update') - .orderBy(accounts.id); - - if (rows.length !== 2) { - throw new Error('account_missing'); - } + await withTxnRetry(() => + db.transaction(async (tx) => { + // Lock both rows in a deterministic order to avoid deadlocks + const rows = await tx + .select() + .from(accounts) + .where(inArray(accounts.id, [fromId, toId])) + .for('update') + .orderBy(accounts.id); + + if (rows.length !== 2) { + throw new Error('account_missing'); + } - const [first, second] = rows; - const fromRow = first.id === fromId ? first : second; - const toRow = first.id === fromId ? second : first; + const [first, second] = rows; + const fromRow = first.id === fromId ? first : second; + const toRow = first.id === fromId ? second : first; - if (fromRow.balance < delta) { - return; // not enough funds, do nothing (same as other backends) - } + if (fromRow.balance < delta) { + return; // not enough funds, do nothing (same as other backends) + } - await tx - .update(accounts) - .set({ balance: fromRow.balance - delta }) - .where(eq(accounts.id, fromId)); + await tx + .update(accounts) + .set({ balance: fromRow.balance - delta }) + .where(eq(accounts.id, fromId)); - await tx - .update(accounts) - .set({ balance: toRow.balance + delta }) - .where(eq(accounts.id, toId)); - }); + await tx + .update(accounts) + .set({ balance: toRow.balance + delta }) + .where(eq(accounts.id, toId)); + }), + ); } async function rpcGetAccount(args: Record) { diff --git a/templates/keynote-2/convex-app/package.json b/templates/keynote-2/convex-app/package.json index eebdd65afc6..e5997e51d0f 100644 --- a/templates/keynote-2/convex-app/package.json +++ b/templates/keynote-2/convex-app/package.json @@ -7,7 +7,7 @@ }, "dependencies": { "@convex-dev/sharded-counter": "^0.2.0", - "convex": "^1.29.3", + "convex": "^1.37.0", "convex-helpers": "^0.1.0" } } diff --git a/templates/keynote-2/docker-compose-crdb-loadbalancer.yml b/templates/keynote-2/docker-compose-crdb-loadbalancer.yml deleted file mode 100644 index 26c5cfbdcbb..00000000000 --- a/templates/keynote-2/docker-compose-crdb-loadbalancer.yml +++ /dev/null @@ -1,7 +0,0 @@ -services: - crdb-rpc-lb: - image: nginx:stable - ports: - - "4102:4102" - volumes: - - ./nginx-crdb.conf:/etc/nginx/nginx.conf:ro diff --git a/templates/keynote-2/docker-compose-crdb-rpc-server.yml b/templates/keynote-2/docker-compose-crdb-rpc-server.yml deleted file mode 100644 index 6b78dd7ae83..00000000000 --- a/templates/keynote-2/docker-compose-crdb-rpc-server.yml +++ /dev/null @@ -1,14 +0,0 @@ -services: - crdb-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] - ports: - - "5001:5001" - environment: - CRDB_URL: ${CRDB_URL} # Point to remote CRDB cluster - CRDB_RPC_PORT: "5001" - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - diff --git a/templates/keynote-2/docker-compose-linux-raid-crdb.yml b/templates/keynote-2/docker-compose-linux-raid-crdb.yml deleted file mode 100644 index 08310ec21ed..00000000000 --- a/templates/keynote-2/docker-compose-linux-raid-crdb.yml +++ /dev/null @@ -1,209 +0,0 @@ -services: - pg: - image: postgres:16 - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - ports: - - "5432:5432" - command: > # faster setup - -c fsync=on - -c synchronous_commit=off - -c shared_buffers=8GB - -c work_mem=64MB - -c max_connections=10000 - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] - interval: 2s - timeout: 2s - retries: 15 - volumes: - - /mnt/local-ssd/pg_data:/var/lib/postgresql/data - network_mode: host - - pg-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/postgres-rpc-server.ts"] - ports: - - "4101:4101" - environment: - PG_URL: ${PG_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - depends_on: - pg: - condition: service_healthy - network_mode: host - - # crdb-rpc-1: - # build: - # context: . - # dockerfile: Dockerfile.rpc - # command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] - # environment: - # CRDB_URL: ${CRDB_URL} - # CRDB_RPC_PORT: "5001" - # SEED_ACCOUNTS: ${SEED_ACCOUNTS} - # SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - # network_mode: host - # - # crdb-rpc-2: - # build: - # context: . - # dockerfile: Dockerfile.rpc - # command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] - # environment: - # CRDB_URL: ${CRDB_URL} - # CRDB_RPC_PORT: "5002" - # SEED_ACCOUNTS: ${SEED_ACCOUNTS} - # SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - # network_mode: host - # - # crdb-rpc-3: - # build: - # context: . - # dockerfile: Dockerfile.rpc - # command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] - # environment: - # CRDB_URL: ${CRDB_URL} - # CRDB_RPC_PORT: "5003" - # SEED_ACCOUNTS: ${SEED_ACCOUNTS} - # SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - # network_mode: host - # - # crdb-rpc-lb: - # image: nginx:stable - # volumes: - # - ./nginx-crdb.conf:/etc/nginx/nginx.conf:ro - # depends_on: - # crdb-rpc-1: - # condition: service_started - # crdb-rpc-2: - # condition: service_started - # crdb-rpc-3: - # condition: service_started - # network_mode: host - - sqlite-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/sqlite-rpc-server.ts"] - ports: - - "4103:4103" - environment: - SQLITE_FILE: /data/accounts.sqlite - SQLITE_MODE: ${SQLITE_MODE} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - volumes: - - /mnt/local-ssd/sqlite_data:/data - network_mode: host - - bun-rpc: - build: - context: . - dockerfile: Dockerfile.bun - ports: - - "4001:4001" - environment: - BUN_PG_URL: ${BUN_PG_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - depends_on: - pg: - condition: service_healthy - network_mode: host - - supabase-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: [ "pnpm", "tsx", "src/rpc-servers/supabase-rpc-server.ts" ] - ports: - - "4106:4106" - environment: - SUPABASE_DB_URL: ${SUPABASE_DB_URL} - PG_RPC_PORT: "4106" - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - network_mode: host - - planetscale-pg-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: [ "pnpm", "tsx", "src/rpc-servers/postgres-rpc-server.ts" ] - ports: - - "4104:4104" - environment: - PG_URL: ${PLANETSCALE_PG_URL} - PG_RPC_PORT: "4104" - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - network_mode: host - - spacetime: - image: clockworklabs/spacetime:latest - command: start - ports: - - "3000:3000" - volumes: - - /mnt/local-ssd/spacetime_data:/data - network_mode: host - - bench: - build: - context: . - dockerfile: Dockerfile.bench - depends_on: - spacetime: - condition: service_started - pg-rpc: - condition: service_started - sqlite-rpc: - condition: service_started - environment: - USE_DOCKER: ${USE_DOCKER} - PG_URL: ${PG_URL} - CRDB_URL: ${CRDB_URL} - CONVEX_URL: ${CONVEX_URL} - STDB_URL: ${STDB_URL} - STDB_MODULE: ${STDB_MODULE} - STDB_MODULE_PATH: ${STDB_MODULE_PATH} - STDB_CONFIRMED_READS: ${STDB_CONFIRMED_READS} - BUN_URL: ${BUN_URL} - SQLITE_FILE: /data/accounts.sqlite - SQLITE_MODE: ${SQLITE_MODE} - SUPABASE_URL: ${SUPABASE_URL} - SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY} - SUPABASE_DB_URL: ${SUPABASE_DB_URL} - PG_RPC_URL: ${PG_RPC_URL} - CRDB_RPC_URL: ${CRDB_RPC_URL} - SQLITE_RPC_URL: ${SQLITE_RPC_URL} - SUPABASE_RPC_URL: ${SUPABASE_RPC_URL} - PLANETSCALE_RPC_URL: ${PLANETSCALE_RPC_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - CONVEX_USE_SHARDED_COUNTER: ${CONVEX_USE_SHARDED_COUNTER} - VERIFY: ${VERIFY} - volumes: - - /mnt/local-ssd/sqlite_data:/data - - ./runs:/app/runs - command: ["--seconds", "5", "--concurrency", "50", "--alpha", "1.5", "--connectors", "sqlite"] - network_mode: host - - sqlite-seed: - build: - context: . - dockerfile: Dockerfile.sqlite-seed - environment: - SQLITE_FILE: /data/accounts.sqlite - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - volumes: - - /mnt/local-ssd/sqlite_data:/data - network_mode: host diff --git a/templates/keynote-2/docker-compose-linux-raid.yml b/templates/keynote-2/docker-compose-linux-raid.yml deleted file mode 100644 index d03fc7339ee..00000000000 --- a/templates/keynote-2/docker-compose-linux-raid.yml +++ /dev/null @@ -1,330 +0,0 @@ -services: - pg: - image: postgres:16 - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - ports: - - "5432:5432" - # command: > - # -c fsync=on - # -c synchronous_commit=on - # -c shared_buffers=2GB - # -c work_mem=64MB - # -c max_connections=1000 - command: > # faster setup - -c fsync=on - -c synchronous_commit=off - -c shared_buffers=8GB - -c work_mem=64MB - -c max_connections=10000 - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] - interval: 2s - timeout: 2s - retries: 15 - volumes: - - /mnt/local-ssd/pg_data:/var/lib/postgresql/data - network_mode: host - - crdb: - image: cockroachdb/cockroach:latest - command: start-single-node --insecure --max-sql-memory=.25 --cache=.5 - ports: - - "26257:26257" - - "8082:8080" - healthcheck: - test: ["CMD", "/cockroach/cockroach", "node", "status", "--insecure"] - interval: 2s - timeout: 2s - retries: 15 - volumes: - - /mnt/local-ssd/crdb_data:/cockroach/cockroach-data - network_mode: host - -# crdb: -# image: cockroachdb/cockroach:latest -# command: > -# start --insecure -# --max-sql-memory=.25 --cache=.3 -# --listen-addr=0.0.0.0:36257 -# --http-addr=0.0.0.0:18080 -# --advertise-addr=127.0.0.1:36257 -# --join=127.0.0.1:36257,127.0.0.1:36258,127.0.0.1:36259 -# healthcheck: -# test: ["CMD-SHELL", "/cockroach/cockroach sql --insecure --host=127.0.0.1:36257 -e 'SELECT 1' >/dev/null 2>&1"] -# interval: 2s -# timeout: 2s -# retries: 60 -# volumes: -# - /mnt/local-ssd/crdb_data:/cockroach/cockroach-data -# network_mode: host -# -# crdb-node2: -# image: cockroachdb/cockroach:latest -# command: > -# start --insecure -# --max-sql-memory=.25 --cache=.3 -# --listen-addr=0.0.0.0:36258 -# --http-addr=0.0.0.0:18081 -# --advertise-addr=127.0.0.1:36258 -# --join=127.0.0.1:36257,127.0.0.1:36258,127.0.0.1:36259 -# volumes: -# - /mnt/local-ssd/crdb_data2:/cockroach/cockroach-data -# network_mode: host -# -# crdb-node3: -# image: cockroachdb/cockroach:latest -# command: > -# start --insecure -# --max-sql-memory=.25 --cache=.3 -# --listen-addr=0.0.0.0:36259 -# --http-addr=0.0.0.0:18082 -# --advertise-addr=127.0.0.1:36259 -# --join=127.0.0.1:36257,127.0.0.1:36258,127.0.0.1:36259 -# volumes: -# - /mnt/local-ssd/crdb_data3:/cockroach/cockroach-data -# network_mode: host -# -# crdb-init: -# image: cockroachdb/cockroach:latest -# depends_on: -# crdb: -# condition: service_healthy -# crdb-node2: -# condition: service_started -# crdb-node3: -# condition: service_started -# network_mode: host -# restart: "no" -# entrypoint: [ "sh", "-c" ] -# command: > -# cockroach init --insecure --host=127.0.0.1:36257 || true && -# cockroach sql --insecure --host=127.0.0.1:36257 -e 'CREATE DATABASE IF NOT EXISTS bench;' - - pg-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/postgres-rpc-server.ts"] - ports: - - "4101:4101" - environment: - PG_URL: ${PG_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - depends_on: - pg: - condition: service_healthy - network_mode: host - -# crdb-rpc-1: -# build: -# context: . -# dockerfile: Dockerfile.rpc -# command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] -# environment: -# CRDB_URL: ${CRDB_URL} -# CRDB_RPC_PORT: "5001" -# SEED_ACCOUNTS: ${SEED_ACCOUNTS} -# SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} -# network_mode: host -# -# crdb-rpc-2: -# build: -# context: . -# dockerfile: Dockerfile.rpc -# command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] -# environment: -# CRDB_URL: ${CRDB_URL} -# CRDB_RPC_PORT: "5002" -# SEED_ACCOUNTS: ${SEED_ACCOUNTS} -# SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} -# network_mode: host -# -# crdb-rpc-3: -# build: -# context: . -# dockerfile: Dockerfile.rpc -# command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] -# environment: -# CRDB_URL: ${CRDB_URL} -# CRDB_RPC_PORT: "5003" -# SEED_ACCOUNTS: ${SEED_ACCOUNTS} -# SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} -# network_mode: host -# -# crdb-rpc-lb: -# image: nginx:stable -# volumes: -# - ./nginx-crdb.conf:/etc/nginx/nginx.conf:ro -# depends_on: -# crdb-rpc-1: -# condition: service_started -# crdb-rpc-2: -# condition: service_started -# crdb-rpc-3: -# condition: service_started -# network_mode: host - - crdb-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/cockroach-rpc-server.ts"] - ports: - - "4102:4102" - environment: - CRDB_URL: ${CRDB_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - depends_on: - crdb: - condition: service_healthy -# crdb-init: -# condition: service_completed_successfully - network_mode: host - - sqlite-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: ["pnpm", "tsx", "src/rpc-servers/sqlite-rpc-server.ts"] - ports: - - "4103:4103" - environment: - SQLITE_FILE: /data/accounts.sqlite - SQLITE_MODE: ${SQLITE_MODE} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - volumes: - - /mnt/local-ssd/sqlite_data:/data - network_mode: host - - bun-rpc: - build: - context: . - dockerfile: Dockerfile.bun - ports: - - "4001:4001" - environment: - BUN_PG_URL: ${BUN_PG_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - depends_on: - pg: - condition: service_healthy - network_mode: host - - supabase-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: [ "pnpm", "tsx", "src/rpc-servers/supabase-rpc-server.ts" ] - ports: - - "4106:4106" - environment: - SUPABASE_DB_URL: ${SUPABASE_DB_URL} - PG_RPC_PORT: "4106" - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - network_mode: host - - planetscale-pg-rpc: - build: - context: . - dockerfile: Dockerfile.rpc - command: [ "pnpm", "tsx", "src/rpc-servers/postgres-rpc-server.ts" ] - ports: - - "4104:4104" - environment: - PG_URL: ${PLANETSCALE_PG_URL} - PG_RPC_PORT: "4104" - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - network_mode: host - - spacetime: - image: clockworklabs/spacetime:latest - command: start - ports: - - "3000:3000" - volumes: - - /mnt/local-ssd/spacetime_data:/data - network_mode: host - # healthcheck: - # test: ["CMD-SHELL", "curl -fsS http://localhost:3000/health || exit 1"] - # interval: 2s - # timeout: 2s - # retries: 15 - - bench: - build: - context: . - dockerfile: Dockerfile.bench - depends_on: - pg: - condition: service_healthy - crdb: - condition: service_healthy -# crdb-init: -# condition: service_completed_successfully - spacetime: - condition: service_started - pg-rpc: - condition: service_started -# crdb-rpc-lb: -# condition: service_started -# crdb-rpc-1: -# condition: service_started -# crdb-rpc-2: -# condition: service_started -# crdb-rpc-3: -# condition: service_started - sqlite-rpc: - condition: service_started - environment: - USE_DOCKER: ${USE_DOCKER} - PG_URL: ${PG_URL} - CRDB_URL: ${CRDB_URL} - CONVEX_URL: ${CONVEX_URL} - STDB_URL: ${STDB_URL} - STDB_MODULE: ${STDB_MODULE} - STDB_MODULE_PATH: ${STDB_MODULE_PATH} - STDB_CONFIRMED_READS: ${STDB_CONFIRMED_READS} - BUN_URL: ${BUN_URL} - SQLITE_FILE: /data/accounts.sqlite - SQLITE_MODE: ${SQLITE_MODE} - SUPABASE_URL: ${SUPABASE_URL} - SUPABASE_ANON_KEY: ${SUPABASE_ANON_KEY} - SUPABASE_DB_URL: ${SUPABASE_DB_URL} - PG_RPC_URL: ${PG_RPC_URL} - CRDB_RPC_URL: ${CRDB_RPC_URL} - SQLITE_RPC_URL: ${SQLITE_RPC_URL} - SUPABASE_RPC_URL: ${SUPABASE_RPC_URL} - PLANETSCALE_RPC_URL: ${PLANETSCALE_RPC_URL} - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - CONVEX_USE_SHARDED_COUNTER: ${CONVEX_USE_SHARDED_COUNTER} - VERIFY: ${VERIFY} - BENCH_PIPELINED: ${BENCH_PIPELINED} - MAX_INFLIGHT_PER_WORKER: ${MAX_INFLIGHT_PER_WORKER} - ENABLE_RPC_SERVERS: ${ENABLE_RPC_SERVERS} - volumes: - - /mnt/local-ssd/sqlite_data:/data - - ./runs:/app/runs - command: ["--seconds", "5", "--concurrency", "50", "--alpha", "1.5", "--connectors", "sqlite"] - network_mode: host - - sqlite-seed: - build: - context: . - dockerfile: Dockerfile.sqlite-seed - environment: - SQLITE_FILE: /data/accounts.sqlite - SEED_ACCOUNTS: ${SEED_ACCOUNTS} - SEED_INITIAL_BALANCE: ${SEED_INITIAL_BALANCE} - volumes: - - /mnt/local-ssd/sqlite_data:/data - network_mode: host diff --git a/templates/keynote-2/docker-compose.yml b/templates/keynote-2/docker-compose.yml index e86735fd13d..d4d07dd8ab6 100644 --- a/templates/keynote-2/docker-compose.yml +++ b/templates/keynote-2/docker-compose.yml @@ -7,19 +7,7 @@ POSTGRES_DB: ${POSTGRES_DB} ports: - "5432:5432" - # command: > - # -c fsync=on - # -c synchronous_commit=on - # -c shared_buffers=2GB - # -c work_mem=64MB - # -c max_connections=1000 - command: > # faster setup - -c fsync=on - -c synchronous_commit=off - -c shared_buffers=8GB - -c work_mem=64MB - -c max_connections=10000 - -c default_transaction_isolation=serializable + command: -c default_transaction_isolation=serializable healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] interval: 2s @@ -31,7 +19,7 @@ crdb: image: cockroachdb/cockroach:latest - command: start-single-node --insecure --max-sql-memory=.25 --cache=.5 + command: start-single-node --insecure ports: - "26257:26257" - "8082:8080" diff --git a/templates/keynote-2/package.json b/templates/keynote-2/package.json index 05157e7d59e..e7278fdb908 100644 --- a/templates/keynote-2/package.json +++ b/templates/keynote-2/package.json @@ -20,7 +20,7 @@ "@types/pg": "^8.15.6", "@types/sql.js": "^1.4.9", "bun-types": "^1.3.2", - "convex": "^1.29.0", + "convex": "^1.37.0", "dotenv": "^17.2.3", "node-gyp": "^12.1.0", "pg": "^8.16.3", diff --git a/templates/keynote-2/scripts/README.md b/templates/keynote-2/scripts/README.md new file mode 100644 index 00000000000..643147047f4 --- /dev/null +++ b/templates/keynote-2/scripts/README.md @@ -0,0 +1,48 @@ +# Bench scripts + +Helpers for running the keynote benchmark on a single host. + +| Script | What it does | +| --- | --- | +| `start-bench.sh` | Bring up every backend service (sqlite-rpc, postgres-rpc, bun-rpc, cockroach + cockroach-rpc, supabase-rpc, convex local) in its own tmux window inside session `bench`. | +| `stop-bench.sh` | Kill every foreground bench process and the tmux session. Leaves Postgres (systemd) and Supabase (Docker) running. | +| `check-bench.sh` | Health-check each service with a single HTTP call. | +| `bench-stats.py` | Read `runs/test-1-*.json` and emit a TSV with aggregate, steady-state, tail-window, and time-series stats. Detects collapse/death points. | +| `plot-bench.py` | Read the `timeSeries` field from `runs/test-1-*.json` and produce per-alpha TPS + latency-percentile charts. Requires matplotlib. | + +Sweep orchestration (multiple alphas, multiple runs, optional state reset) is now built into `pnpm run bench` itself. See the project README's "Test Command" section. + +## Typical flow + +```bash +# one-time prerequisites +sudo systemctl start postgresql # Postgres as system service +supabase start # Supabase Docker stack +sudo -u postgres psql -c "CREATE DATABASE bun_bench;" + +# bring services up +scripts/start-bench.sh +tmux attach -t bench # poke around if needed +scripts/check-bench.sh # confirm all green + +# seed once, then sweep alphas and connectors +pnpm run prep +pnpm run bench --alpha 0,1.5 --connectors postgres_rpc,bun --seconds 300 + +# multi-run sweep with auto-reset between alphas +pnpm run bench --alpha 0,1.5 --connectors postgres_rpc --seconds 300 --runs 3 --prep-between-alphas + +# stats + plots from the per-run JSONs +python3 scripts/bench-stats.py --runs-dir runs +python3 scripts/plot-bench.py 0 +python3 scripts/plot-bench.py 1.5 + +# tear down +scripts/stop-bench.sh +``` + +## Notes + +- `bench-stats.py` and `plot-bench.py` glob `runs/test-1-*.json`. To keep separate sweeps from mixing, organize JSONs into subdirectories per sweep and point `--runs-dir` at each one. +- `plot-bench.py` requires the `timeSeries` field on each run, added by the current `core/runner.ts`. Older JSON files without that field are silently skipped. +- All scripts resolve paths relative to the script file, so the checkout can live anywhere. diff --git a/templates/keynote-2/scripts/bench-stats.py b/templates/keynote-2/scripts/bench-stats.py new file mode 100644 index 00000000000..a67b977028d --- /dev/null +++ b/templates/keynote-2/scripts/bench-stats.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Compute detailed stats from each test-1-*.json run in a runs-dir. + +Pulls out: + - aggregate metrics (already in res.*) + - steady-state window stats (t >= --warmup-sec, default 30) + - tail window stats (last --tail-sec seconds, default 30) + - time-series shape: tps min/max/mean/median/stdev, stability (CV) + - collapse detection: first second where TPS drops below 10% of peak + - death detection: first second where TPS=0 and stays 0 for 30s + +Usage: + python3 bench-stats.py [--runs-dir DIR] [--warmup-sec N] [--tail-sec N] [--out FILE] + +Examples: + python3 bench-stats.py --runs-dir D:/keynote-2-runs + python3 bench-stats.py --runs-dir D:/keynote-2-runs --warmup-sec 60 --out stats.tsv +""" +import argparse +import json +import math +import statistics +import sys +from pathlib import Path + +DEFAULT_RUNS_DIR = Path(__file__).resolve().parent.parent / "runs" + + +def load_run(path): + data = json.loads(path.read_text()) + r = data["results"][0] + return { + "path": path.name, + "system": r.get("system", r.get("file", "?").replace(".ts", "")), + "alpha": data["alpha"], + "seconds": data["seconds"], + "concurrency": data["concurrency"], + "agg": r["res"], + "ts": r["res"].get("timeSeries", []), + } + + +def safe_div(a, b): + return a / b if b else 0.0 + + +def window_stats(ts, t_min, t_max=None): + """Return aggregate stats for time-series points where t_min <= tSec < t_max.""" + pts = [p for p in ts if p["tSec"] >= t_min and (t_max is None or p["tSec"] < t_max)] + if not pts: + return None + + samples = sum(p["samples"] for p in pts) + duration = sum(1 for _ in pts) # one point per second + tps_values = [p["tps"] for p in pts] + nonzero_tps = [v for v in tps_values if v > 0] + p50_values = [p["p50_ms"] for p in pts if p["samples"] > 0] + p99_values = [p["p99_ms"] for p in pts if p["samples"] > 0] + + return { + "samples": samples, + "duration_s": duration, + "tps_mean": safe_div(samples, duration), + "tps_min": min(tps_values) if tps_values else 0, + "tps_max": max(tps_values) if tps_values else 0, + "tps_median": statistics.median(tps_values) if tps_values else 0, + "tps_stdev": statistics.stdev(tps_values) if len(tps_values) > 1 else 0, + "tps_cv_pct": ( + (statistics.stdev(tps_values) / statistics.mean(tps_values) * 100) + if len(tps_values) > 1 and statistics.mean(tps_values) > 0 + else 0 + ), + "zero_seconds": sum(1 for v in tps_values if v == 0), + "p50_ms_median": statistics.median(p50_values) if p50_values else 0, + "p99_ms_median": statistics.median(p99_values) if p99_values else 0, + "p99_ms_max": max(p99_values) if p99_values else 0, + } + + +def find_collapse(ts, threshold_pct=10): + """First tSec where TPS drops below threshold_pct% of peak. + Returns None if it never collapses.""" + if not ts: + return None + peak = max(p["tps"] for p in ts) + if peak == 0: + return None + threshold = peak * threshold_pct / 100 + # require sustained drop (3 consecutive seconds below threshold) + streak = 0 + for p in ts: + if p["tps"] < threshold: + streak += 1 + if streak >= 3: + return p["tSec"] - 2 # first second of the streak + else: + streak = 0 + return None + + +def find_death(ts, hold_sec=30): + """First tSec where TPS hits 0 and stays 0 for hold_sec seconds. + Returns None if never dies.""" + streak = 0 + for p in ts: + if p["tps"] == 0: + streak += 1 + if streak >= hold_sec: + return p["tSec"] - hold_sec + 1 + else: + streak = 0 + return None + + +def fmt(v): + if v is None: + return "" + if isinstance(v, float): + if math.isnan(v) or math.isinf(v): + return "" + if abs(v) >= 100: + return f"{v:.1f}" + if abs(v) >= 1: + return f"{v:.2f}" + return f"{v:.4f}" + return str(v) + + +COLUMNS = [ + "system", + "alpha", + "duration_s", + "concurrency", + # aggregate (whole run) + "agg_tps", + "agg_samples", + "agg_p50_ms", + "agg_p95_ms", + "agg_p99_ms", + "agg_collision_rate", + # steady-state (after warmup) + "ss_tps_mean", + "ss_tps_median", + "ss_tps_stdev", + "ss_tps_cv_pct", + "ss_p50_ms", + "ss_p99_ms", + "ss_zero_secs", + # tail (last N seconds) + "tail_tps_mean", + "tail_p50_ms", + "tail_p99_ms", + # time-series shape + "ts_tps_min", + "ts_tps_max", + "ts_p99_max", + # collapse / death + "collapse_at_s", + "death_at_s", + # source + "file", +] + + +def row_for_run(run, warmup_sec, tail_sec): + agg = run["agg"] + ts = run["ts"] + duration = run["seconds"] + + ss = window_stats(ts, warmup_sec) if ts else None + tail_start = duration - tail_sec + tail = window_stats(ts, tail_start) if ts else None + full = window_stats(ts, 0) if ts else None + + return { + "system": run["system"], + "alpha": run["alpha"], + "duration_s": duration, + "concurrency": run["concurrency"], + "agg_tps": agg.get("tps"), + "agg_samples": agg.get("samples"), + "agg_p50_ms": agg.get("p50_ms"), + "agg_p95_ms": agg.get("p95_ms"), + "agg_p99_ms": agg.get("p99_ms"), + "agg_collision_rate": agg.get("collision_rate"), + "ss_tps_mean": ss["tps_mean"] if ss else None, + "ss_tps_median": ss["tps_median"] if ss else None, + "ss_tps_stdev": ss["tps_stdev"] if ss else None, + "ss_tps_cv_pct": ss["tps_cv_pct"] if ss else None, + "ss_p50_ms": ss["p50_ms_median"] if ss else None, + "ss_p99_ms": ss["p99_ms_median"] if ss else None, + "ss_zero_secs": ss["zero_seconds"] if ss else None, + "tail_tps_mean": tail["tps_mean"] if tail else None, + "tail_p50_ms": tail["p50_ms_median"] if tail else None, + "tail_p99_ms": tail["p99_ms_median"] if tail else None, + "ts_tps_min": full["tps_min"] if full else None, + "ts_tps_max": full["tps_max"] if full else None, + "ts_p99_max": full["p99_ms_max"] if full else None, + "collapse_at_s": find_collapse(ts), + "death_at_s": find_death(ts), + "file": run["path"], + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--runs-dir", type=Path, default=DEFAULT_RUNS_DIR) + parser.add_argument("--warmup-sec", type=int, default=30, + help="Seconds to skip before computing steady-state stats") + parser.add_argument("--tail-sec", type=int, default=30, + help="Tail window size for last-N-seconds stats") + parser.add_argument("--out", type=Path, default=None, + help="Write TSV here. If a bare filename, lands in --runs-dir. " + "If omitted, defaults to /stats.tsv.") + args = parser.parse_args() + + files = sorted(args.runs_dir.glob("test-1-*.json")) + if not files: + print(f"no test-1-*.json files in {args.runs_dir}", file=sys.stderr) + sys.exit(1) + + rows = [row_for_run(load_run(p), args.warmup_sec, args.tail_sec) for p in files] + + # sort by (alpha, system) for readability + rows.sort(key=lambda r: (r["alpha"], r["system"])) + + header = "\t".join(COLUMNS) + body = "\n".join("\t".join(fmt(r[c]) for c in COLUMNS) for r in rows) + output = header + "\n" + body + "\n" + + # Resolve output path: default to /stats.tsv; + # bare filenames land in runs-dir; absolute paths are used as-is. + if args.out is None: + out_path = args.runs_dir / "stats.tsv" + elif args.out.parent == Path("."): + out_path = args.runs_dir / args.out + else: + out_path = args.out + + out_path.write_text(output) + print(output) + print(f"\nwrote {out_path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/templates/keynote-2/scripts/check-bench.sh b/templates/keynote-2/scripts/check-bench.sh new file mode 100644 index 00000000000..fc3e63fe6cb --- /dev/null +++ b/templates/keynote-2/scripts/check-bench.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Health-check every backend the bench needs. +# Prints one line per service. Returns 0 even on failures — visual inspection. + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." &> /dev/null && pwd)" +CONVEX_URL=$(grep '^CONVEX_URL=' "$ROOT/convex-app/.env.local" 2>/dev/null | cut -d= -f2) + +checks=( + "sqlite_rpc|http://127.0.0.1:4103/rpc|{\"name\":\"health\",\"args\":{}}" + "postgres_rpc|http://127.0.0.1:4101/rpc|{\"name\":\"health\",\"args\":{}}" + "cockroach_rpc|http://127.0.0.1:4102/rpc|{\"name\":\"health\",\"args\":{}}" + "bun|http://127.0.0.1:4001/rpc|{\"name\":\"health\",\"args\":{}}" + "supabase_rpc|http://127.0.0.1:4106/rpc|{\"name\":\"health\",\"args\":{}}" +) + +for c in "${checks[@]}"; do + IFS='|' read -r name url body <<<"$c" + out=$(curl -s --max-time 3 -X POST "$url" -H 'content-type: application/json' -d "$body" 2>&1) + printf "%-15s %s\n" "$name" "${out:-NO RESPONSE}" +done + +echo +printf "%-15s " "convex" +if [ -n "$CONVEX_URL" ]; then + curl -s --max-time 3 "$CONVEX_URL/instance_name" 2>&1 || echo "(no response)" + echo +else + echo "URL not set in convex-app/.env.local" +fi diff --git a/templates/keynote-2/scripts/plot-bench.py b/templates/keynote-2/scripts/plot-bench.py new file mode 100644 index 00000000000..6195bd28ab7 --- /dev/null +++ b/templates/keynote-2/scripts/plot-bench.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Plot TPS and latency-percentile over time for a single alpha across connectors. + +Reads timeSeries arrays from runs/test-1-*.json (added by core/runner.ts) +and emits a stacked TPS + latency chart per alpha. + +Usage: + python3 plot-bench.py [alpha] [outfile] [--runs-dir DIR] [--exclude conn1,conn2] + [--latency p50|p95|p99] [--metric both|tps|latency] + [--tps-scale linear|log|symlog] [--latency-scale log|linear|symlog] + +Examples: + python3 plot-bench.py 0 + python3 plot-bench.py 1.5 + python3 plot-bench.py 1.5 contended.png + python3 plot-bench.py 1.5 no-stdb.png --exclude spacetimedb + python3 plot-bench.py 1.5 chart.png --runs-dir D:/keynote-2-runs --latency p95 + python3 plot-bench.py 1.5 chart.png --tps-scale log --latency-scale linear +""" +import argparse +import json +import sys +from pathlib import Path +import matplotlib.pyplot as plt + +DEFAULT_RUNS_DIR = Path(__file__).resolve().parent.parent / "runs" + + +def load_run(path): + data = json.loads(path.read_text()) + r = data["results"][0] + return { + "path": path.name, + "connector": r["file"].replace(".ts", ""), + "alpha": data["alpha"], + "ts": r["res"].get("timeSeries", []), + } + + +def plot(runs, alpha, outfile, exclude=None, latency="p99", metric="both", + tps_scale="linear", latency_scale="log"): + matched = [r for r in runs if r["alpha"] == alpha and r["ts"]] + if exclude: + matched = [r for r in matched if r["connector"] not in exclude] + + if not matched: + print(f"no runs with timeSeries data found at alpha={alpha}", file=sys.stderr) + sys.exit(1) + + latency_key = f"{latency}_ms" + + if metric == "both": + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(11, 8), sharex=True) + axes = [(ax1, "tps", "TPS"), (ax2, latency_key, f"{latency} latency (ms)")] + elif metric == "tps": + fig, ax1 = plt.subplots(1, 1, figsize=(11, 5)) + axes = [(ax1, "tps", "TPS")] + elif metric == "latency": + fig, ax1 = plt.subplots(1, 1, figsize=(11, 5)) + axes = [(ax1, latency_key, f"{latency} latency (ms)")] + else: + raise ValueError(f"unknown metric: {metric}") + + # one line per run; group by connector for legend de-dup + seen_connectors = {} + for r in matched: + ts = r["ts"] + x = [p["tSec"] for p in ts] + + label = r["connector"] + if label in seen_connectors: + label = None + else: + seen_connectors[r["connector"]] = True + + for ax, key, _ in axes: + ax.plot(x, [p[key] for p in ts], label=label, linewidth=2, alpha=0.85) + + contention = "uncontended" if alpha == 0 else f"alpha={alpha}" + title = f"alpha={alpha} ({contention})" + if exclude: + title += f" (excluded: {','.join(exclude)})" + + for i, (ax, key, ylabel) in enumerate(axes): + ax.set_ylabel(ylabel) + ax.legend(loc="upper right" if key == "tps" else "upper left") + ax.grid(True, alpha=0.3, which="both") + + scale = tps_scale if key == "tps" else latency_scale + if scale == "log": + ax.set_yscale("log") + elif scale == "symlog": + ax.set_yscale("symlog", linthresh=1) + # "linear" leaves the matplotlib default + + if i == 0: + ax.set_title(title) + + axes[-1][0].set_xlabel("Time (s)") + + plt.tight_layout() + plt.savefig(outfile, dpi=120) + print(f"wrote {outfile} ({len(matched)} runs, metric={metric}, latency={latency})") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("alpha", nargs="?", type=float, default=0) + parser.add_argument("outfile", nargs="?", default=None) + parser.add_argument("--runs-dir", type=Path, default=DEFAULT_RUNS_DIR, + help="directory containing test-1-*.json files") + parser.add_argument("--exclude", default="", + help="comma-separated connectors to skip") + parser.add_argument("--latency", choices=["p50", "p95", "p99"], default="p99", + help="which latency percentile to plot") + parser.add_argument("--metric", choices=["both", "tps", "latency"], default="both", + help="show TPS only, latency only, or both panels") + parser.add_argument("--tps-scale", choices=["linear", "log", "symlog"], default="linear", + help="y-axis scale for the TPS panel") + parser.add_argument("--latency-scale", choices=["log", "linear", "symlog"], default="log", + help="y-axis scale for the latency panel") + args = parser.parse_args() + + # If outfile is just a filename (not a path), put it in the runs dir. + if args.outfile: + outfile_path = Path(args.outfile) + if outfile_path.parent == Path("."): + outfile_path = args.runs_dir / outfile_path + else: + outfile_path = args.runs_dir / f"bench-alpha{args.alpha}-{args.metric}-{args.latency}.png" + + exclude = [c.strip() for c in args.exclude.split(",") if c.strip()] + + runs = [load_run(p) for p in sorted(args.runs_dir.glob("test-1-*.json"))] + plot( + runs, + args.alpha, + str(outfile_path), + exclude=exclude, + latency=args.latency, + metric=args.metric, + tps_scale=args.tps_scale, + latency_scale=args.latency_scale, + ) diff --git a/templates/keynote-2/scripts/start-bench.sh b/templates/keynote-2/scripts/start-bench.sh new file mode 100644 index 00000000000..c02d13292b1 --- /dev/null +++ b/templates/keynote-2/scripts/start-bench.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Start every backend service we bench in its own tmux window inside session "bench". +# Idempotent — re-running kills+recreates each named window. +# +# Prerequisites the script doesn't manage: +# - Postgres running as a system service (sudo systemctl start postgresql) +# - Supabase Docker stack up (supabase start) +# - bun_bench database created in Postgres +# - .env populated (CONVEX_URL etc.) + +set -uo pipefail + +SESSION=bench +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." &> /dev/null && pwd)" + +tmux has-session -t "$SESSION" 2>/dev/null || tmux new-session -d -s "$SESSION" -n bench + +start_window() { + local name=$1; shift + local cmd="$*" + tmux kill-window -t "${SESSION}:${name}" 2>/dev/null || true + tmux new-window -a -t "${SESSION}:" -n "${name}" \ + "bash -c '${cmd}; rc=\$?; echo; echo \"[${name} exited rc=\$rc]\"; read'" +} + +start_window sqlite-rpc "cd $ROOT && pnpm tsx src/rpc-servers/sqlite-rpc-server.ts" +start_window postgres-rpc "cd $ROOT && pnpm tsx src/rpc-servers/postgres-rpc-server.ts" +start_window bun-rpc "cd $ROOT && bun run bun/bun-server.ts" + +start_window cockroach "mkdir -p /tmp/crdb-data && cockroach start-single-node --insecure --listen-addr=127.0.0.1:26257 --http-addr=127.0.0.1:8081 --store=/tmp/crdb-data" +sleep 5 +start_window cockroach-rpc "cd $ROOT && pnpm tsx src/rpc-servers/cockroach-rpc-server.ts" + +start_window supabase-rpc "cd $ROOT && pnpm tsx src/rpc-servers/supabase-rpc-server.ts" +start_window convex "cd $ROOT/convex-app && npx convex dev --local" + +# PlanetScale RPC server: reuses postgres-rpc-server.ts but with PLANETSCALE_PG_URL +# and a different port (4104) to avoid colliding with local postgres-rpc on 4101. +# Only starts if PLANETSCALE_PG_URL is set in .env. +if [ -f "$ROOT/.env" ] && grep -q '^PLANETSCALE_PG_URL=postgres' "$ROOT/.env"; then + start_window planetscale-rpc "cd $ROOT && set -a && . ./.env && set +a && PG_URL=\"\$PLANETSCALE_PG_URL\" PG_RPC_PORT=4104 pnpm tsx src/rpc-servers/postgres-rpc-server.ts" +fi + +echo "All windows started in tmux session '$SESSION'." +echo "Attach: tmux attach -t $SESSION (then Ctrl+B w to browse)" diff --git a/templates/keynote-2/scripts/stop-bench.sh b/templates/keynote-2/scripts/stop-bench.sh new file mode 100644 index 00000000000..c528434107d --- /dev/null +++ b/templates/keynote-2/scripts/stop-bench.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Kill every foreground bench process and the tmux session. +# Leaves Postgres (systemd) and Supabase (Docker) running — those have their own lifecycles. + +pkill -f sqlite-rpc-server 2>/dev/null +pkill -f postgres-rpc-server 2>/dev/null +pkill -f cockroach-rpc-server 2>/dev/null +pkill -f supabase-rpc-server 2>/dev/null +pkill -f bun-server 2>/dev/null +pkill -f "convex dev" 2>/dev/null +pkill -f "cockroach start-single-node" 2>/dev/null +tmux kill-session -t bench 2>/dev/null +echo "stopped." diff --git a/templates/keynote-2/src/cli.ts b/templates/keynote-2/src/cli.ts index a5da83b8002..0c17b2e56df 100644 --- a/templates/keynote-2/src/cli.ts +++ b/templates/keynote-2/src/cli.ts @@ -1,5 +1,6 @@ import 'dotenv/config'; -import { readdir, mkdir, writeFile } from 'node:fs/promises'; +import { mkdir, readdir, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; import { CONNECTORS } from './connectors'; import { runOne } from './core/runner'; import type { TestCaseModule } from './tests/types'; @@ -14,7 +15,9 @@ const { seconds, concurrency, accounts, - alpha, + alphas, + runs, + prepBetweenAlphas, connectors, contentionTests, concurrencyTests, @@ -62,7 +65,7 @@ class BenchmarkTester { await new Promise((resolve) => setTimeout(resolve, 1000)); } - const avg = { + const avg: RunResult = { tps: totals.tps / runs, samples: totals.samples / runs, p50_ms: totals.p50_ms / runs, @@ -71,6 +74,9 @@ class BenchmarkTester { collision_ops: totals.collision_ops / runs, collision_count: totals.collision_count / runs, collision_rate: totals.collision_rate / runs, + // timeSeries can't be meaningfully averaged across runs (each run has its + // own t=0..N curve), so the aggregated avg drops it. + timeSeries: [], }; return avg; } @@ -130,15 +136,42 @@ class BenchmarkTester { } } +/** Subprocess `pnpm run prep` to reset DB state. Inherits stdio so output is visible. */ +function runPrep(): Promise { + return new Promise((resolve, reject) => { + const child = spawn('pnpm', ['run', 'prep'], { + stdio: 'inherit', + shell: true, + }); + child.on('close', (code) => { + if (code === 0) resolve(); + else reject(new Error(`prep exited with code ${code}`)); + }); + child.on('error', reject); + }); +} + const testDirUrl = new URL(`./tests/${testName}/`, import.meta.url); const testDirPath = fileURLToPath(testDirUrl); +const runsDir = fileURLToPath(new URL('../runs/', import.meta.url)); + +async function writeRunJson(payload: object, connectorName: string, alpha: number) { + await mkdir(runsDir, { recursive: true }); + const ts = new Date().toISOString().replace(/[:.]/g, '-'); + const outFile = join(runsDir, `${testName}-${connectorName}-a${alpha}-${ts}.json`); + await writeFile(outFile, JSON.stringify(payload, null, 2)); + console.log(`Wrote results to ${outFile}`); + return outFile; +} (async () => { const files = (await readdir(testDirPath)).filter( (f) => (f.endsWith('.ts') || f.endsWith('.js')) && !f.endsWith('.d.ts'), ); - const results: any[] = []; + // Sweep-mode results accumulate into a single combined JSON (legacy behavior). + const sweepResults: any[] = []; + let sweepAlpha: number | null = null; for (const file of files) { const mod = (await import( @@ -151,74 +184,121 @@ const testDirPath = fileURLToPath(testDirUrl); const makeConnector = CONNECTORS[tc.system]; if (!makeConnector) throw new Error(`Unknown connector ${tc.system}`); - const connector = makeConnector(options); - - let res: any; - - const config = { - connector, - scenario: tc.run, - seconds, - accounts, - runtimeConfig: options, - }; - - const tester = new BenchmarkTester(config); - - if (contentionTests) { - res = await tester.contentionTests( - contentionTests.startAlpha, - contentionTests.endAlpha, - contentionTests.step, - contentionTests.concurrency, - ); - } else if (concurrencyTests) { - res = await tester.concurrencyTestsMutiply( - concurrencyTests.startConc, - concurrencyTests.endConc, - concurrencyTests.step, - concurrencyTests.alpha, - ); - } else { - res = await runOne({ + if (contentionTests || concurrencyTests) { + // Sweep modes: one connector, one combined result, accumulate for a single JSON. + const connector = makeConnector(options); + const config = { connector, scenario: tc.run, seconds, - concurrency, accounts, - alpha, runtimeConfig: options, + }; + const tester = new BenchmarkTester(config); + + let res: any; + if (contentionTests) { + res = await tester.contentionTests( + contentionTests.startAlpha, + contentionTests.endAlpha, + contentionTests.step, + contentionTests.concurrency, + ); + } else if (concurrencyTests) { + res = await tester.concurrencyTestsMutiply( + concurrencyTests.startConc, + concurrencyTests.endConc, + concurrencyTests.step, + concurrencyTests.alpha, + ); + sweepAlpha = concurrencyTests.alpha; + } + + sweepResults.push({ + system: connector.name, + label: tc.label ?? file, + file, + seconds, + concurrency, + accounts, + alpha: sweepAlpha ?? alphas[0], + res, }); + console.log(`${file}:`, res); + continue; + } + + // Basic mode: sweep alphas and repeat runs, writing one JSON per + // (connector, alpha, run) tuple. Optionally prep before each alpha. + for (const alpha of alphas) { + if (prepBetweenAlphas) { + console.log(`[bench] prep before ${tc.system} alpha=${alpha}`); + await runPrep(); + } + + for (let r = 0; r < runs; r++) { + // Create the connector fresh per (alpha, run) so that prep-induced + // schema/state changes don't get cached on a stale connector instance. + const connector = makeConnector(options); + const res = await runOne({ + connector, + scenario: tc.run, + seconds, + concurrency, + accounts, + alpha, + runtimeConfig: options, + }); + + const payload = { + test: testName, + seconds, + concurrency, + accounts, + alpha, + run: r + 1, + runs, + config: { + benchPipelined: options.benchPipelined, + maxInflightPerWorker: options.maxInflightPerWorker, + poolMax: options.poolMax, + stdbConfirmedReads: options.stdbConfirmedReads, + stdbCompression: options.stdbCompression, + }, + results: [ + { + system: connector.name, + label: tc.label ?? file, + file, + seconds, + concurrency, + accounts, + alpha, + res, + }, + ], + }; + await writeRunJson(payload, connector.name, alpha); + console.log( + `[bench] ${tc.system} alpha=${alpha} run ${r + 1}/${runs} done`, + ); + } } + } - results.push({ - system: connector.name, - label: tc.label ?? file, - file, + if (sweepResults.length > 0) { + const payload = { + test: testName, seconds, concurrency, accounts, - alpha, - res, - }); - console.log(`${file}:`, res); + alpha: sweepAlpha ?? alphas[0], + results: sweepResults, + }; + await mkdir(runsDir, { recursive: true }); + const ts = new Date().toISOString().replace(/[:.]/g, '-'); + const outFile = join(runsDir, `${testName}-${ts}.json`); + await writeFile(outFile, JSON.stringify(payload, null, 2)); + console.log(`Wrote sweep results to ${outFile}`); } - - const runData = { - test: testName, - seconds, - concurrency, - accounts, - alpha, - results, - }; - const runsDir = fileURLToPath(new URL('../runs/', import.meta.url)); - await mkdir(runsDir, { recursive: true }); - const outFile = join( - runsDir, - `${testName}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`, - ); - await writeFile(outFile, JSON.stringify(runData, null, 2)); - - console.log(`Wrote results to ${outFile}`); })(); diff --git a/templates/keynote-2/src/config.ts b/templates/keynote-2/src/config.ts index d6d30977986..b6a28a52107 100644 --- a/templates/keynote-2/src/config.ts +++ b/templates/keynote-2/src/config.ts @@ -72,7 +72,20 @@ export interface BenchOptions extends SharedRuntimeConfig { testName: string; seconds: number; concurrency: number; - alpha: number; + /** + * Alphas to sweep. The basic-bench code path writes one JSON per (connector, + * alpha, run) tuple. For backward compatibility, a single `--alpha N` argument + * resolves to a single-element array. + */ + alphas: number[]; + /** Number of times to repeat each (connector, alpha) combination. */ + runs: number; + /** + * If true, runs `pnpm run prep` before each (connector, alpha) combination + * to reset DB state. Each repeat run within the same (connector, alpha) uses + * the same prepped state (so inter-run variance is meaningful). + */ + prepBetweenAlphas: boolean; connectors: ConnectorKey[] | null; contentionTests: ContentionTests | null; concurrencyTests: ConcurrencyTests | null; @@ -106,6 +119,7 @@ export type RunnerRuntimeConfig = Pick< | 'maxInflightPerWorker' | 'minOpTimeoutMs' | 'opTimeoutMs' + | 'poolMax' | 'precomputedTransferPairs' | 'tailSlackMs' | 'verifyTransactions' @@ -193,6 +207,23 @@ export function readOptionalBooleanEnv( return parseBooleanLike(raw); } +export function parseAlphaList( + raw: string | number | string[] | undefined, + label: string, +): number[] | undefined { + if (raw === undefined) return undefined; + if (typeof raw === 'number') return [raw]; + + const values = (Array.isArray(raw) ? raw : [raw]) + .flatMap((value) => String(value).split(',')) + .map((value) => value.trim()) + .filter(Boolean); + + if (values.length === 0) return undefined; + + return values.map((value) => parseFiniteNumber(value, label)); +} + export function parseConnectorList( raw: string | string[] | undefined, label: string, @@ -222,7 +253,7 @@ export function getSharedRuntimeDefaults( ): SharedRuntimeConfig { return { accounts: readNumberEnv('SEED_ACCOUNTS', 100_000, env), - initialBalance: readNumberEnv('SEED_INITIAL_BALANCE', 10_000_000, env), + initialBalance: readNumberEnv('SEED_INITIAL_BALANCE', 1_000_000_000, env), stdbUrl: normalizeStdbUrl(readStringEnv('STDB_URL', '127.0.0.1:3000', env)), stdbModule: readStringEnv('STDB_MODULE', 'test-1', env), stdbModulePath: readStringEnv('STDB_MODULE_PATH', './spacetimedb', env), @@ -232,7 +263,7 @@ export function getSharedRuntimeDefaults( ), stdbConfirmedReads: readBooleanEnv('STDB_CONFIRMED_READS', true, env), useDocker: readBooleanEnv('USE_DOCKER', false, env), - poolMax: readNumberEnv('MAX_POOL', 1000, env), + poolMax: readNumberEnv('MAX_POOL', 64, env), bunUrl: readStringEnv('BUN_URL', 'http://127.0.0.1:4000', env), convexUrl: readStringEnv('CONVEX_URL', 'http://127.0.0.1:3210', env), convexDir: readStringEnv('CONVEX_DIR', './convex-app', env), diff --git a/templates/keynote-2/src/connectors/convex.ts b/templates/keynote-2/src/connectors/convex.ts index b6585772fab..349f02216ea 100644 --- a/templates/keynote-2/src/connectors/convex.ts +++ b/templates/keynote-2/src/connectors/convex.ts @@ -3,14 +3,6 @@ import type { RpcConnector } from '../core/connectors.ts'; export default function convex(url: string): RpcConnector { if (!url) throw new Error('CONVEX_URL not set'); - function isWriteConflict(msg: unknown): boolean { - if (typeof msg !== 'string') return false; - return ( - msg.includes('Documents read from or written to the') && - msg.includes('while this mutation was being run') - ); - } - async function queryConvex(path: string, args: any) { const res = await fetch(`${url}/api/query?format=json`, { method: 'POST', @@ -36,53 +28,31 @@ export default function convex(url: string): RpcConnector { } async function mutationConvex(path: string, args: any) { - const MAX_RETRIES = 32; - const BASE_DELAY_MS = 0.1; - const MAX_DELAY_MS = 100; - - for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { - const res = await fetch(`${url}/api/mutation?format=json`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ path, args }), - }); - - let json: any = {}; - try { - json = await res.json(); - } catch {} - - const ok = res.ok && json.status === 'success'; - const msgRaw = - json?.errorMessage ?? - json?.message ?? - `HTTP ${res.status} ${res.statusText}`; - const msg = String(msgRaw); - const writeConflict = isWriteConflict(msg); - - if (ok) { - return json.value; - } + const res = await fetch(`${url}/api/mutation?format=json`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ path, args }), + }); - if (writeConflict && attempt < MAX_RETRIES) { - const base = BASE_DELAY_MS * 2 ** attempt; - const delay = - Math.min(MAX_DELAY_MS, base) + Math.floor(Math.random() * 10); - await new Promise((r) => setTimeout(r, delay)); - continue; - } + let json: any = {}; + try { + json = await res.json(); + } catch {} - throw new Error(`convex mutation ${path} failed: ${msg}`); + if (res.ok && json.status === 'success') { + return json.value; } - throw new Error( - `convex mutation ${path} failed after ${MAX_RETRIES} retries due to write conflicts`, - ); + const msg = + json?.errorMessage ?? + json?.message ?? + `HTTP ${res.status} ${res.statusText}`; + + throw new Error(`convex mutation ${path} failed: ${msg}`); } const root: RpcConnector = { name: 'convex', - maxInflightPerWorker: 16, async open() {}, async close() {}, diff --git a/templates/keynote-2/src/connectors/rpc/cockroach_rpc.ts b/templates/keynote-2/src/connectors/rpc/cockroach_rpc.ts index 01034dc0c89..17d46a49a5c 100644 --- a/templates/keynote-2/src/connectors/rpc/cockroach_rpc.ts +++ b/templates/keynote-2/src/connectors/rpc/cockroach_rpc.ts @@ -46,37 +46,6 @@ export default function cockroach_rpc( return json.result; } - async function callWithRetry( - name: string, - args?: Record, - maxRetries: number = 5, - ) { - let attempts = 0; - while (attempts < maxRetries) { - try { - return await httpCall(name, args); - } catch (err: unknown) { - let errMsg = 'Unknown error'; - if (err instanceof Error) { - errMsg = err.message; - } else if (typeof err === 'string') { - errMsg = err; - } - if ( - errMsg.includes('serialization') || - errMsg.includes('restart transaction') || - errMsg.includes('40001') - ) { - attempts++; - if (attempts >= maxRetries) throw err; - continue; - } - throw err; - } - } - throw new Error('Max retries exceeded'); - } - const root: RpcConnector = { name: 'cockroach_rpc', @@ -107,9 +76,6 @@ export default function cockroach_rpc( }, async call(name: string, args?: Record) { - if (name === 'transfer') { - return callWithRetry(name, args); - } return httpCall(name, args); }, diff --git a/templates/keynote-2/src/connectors/spacetimedb.ts b/templates/keynote-2/src/connectors/spacetimedb.ts index 57f25a9d000..5a6ae50e164 100644 --- a/templates/keynote-2/src/connectors/spacetimedb.ts +++ b/templates/keynote-2/src/connectors/spacetimedb.ts @@ -78,7 +78,6 @@ export function spacetimedb(config: SpacetimeConnectorConfig): ReducerConnector return { name: 'spacetimedb', - maxInflightPerWorker: 128, async open() { try { diff --git a/templates/keynote-2/src/core/connectors.ts b/templates/keynote-2/src/core/connectors.ts index e5be3ef5a80..b259b2ad35c 100644 --- a/templates/keynote-2/src/core/connectors.ts +++ b/templates/keynote-2/src/core/connectors.ts @@ -8,8 +8,6 @@ export interface BaseConnector { } | null>; verify(): Promise; - maxInflightPerWorker?: number; - createWorker?(opts: { index: number; total: number }): Promise; } diff --git a/templates/keynote-2/src/core/runner.ts b/templates/keynote-2/src/core/runner.ts index d363586170f..c38aab33ba0 100644 --- a/templates/keynote-2/src/core/runner.ts +++ b/templates/keynote-2/src/core/runner.ts @@ -125,16 +125,20 @@ export async function runOne({ `[${connector.name}] precomputed ${transferPairs.count} pairs in ${(precomputeElapsedMs / 1000).toFixed(2)}s`, ); - const PIPELINED = benchPipelined ?? !!connector.maxInflightPerWorker; - const MAX_INFLIGHT_PER_WORKER = - maxInflightPerWorker === undefined - ? (connector.maxInflightPerWorker ?? 8) - : maxInflightPerWorker == 0 - ? Infinity - : maxInflightPerWorker; + const PIPELINED = benchPipelined ?? false; + let MAX_INFLIGHT_PER_WORKER = 1; + if (PIPELINED && maxInflightPerWorker === undefined) { + throw new Error( + `[${connector.name}] pipelining is enabled, but max inflight per worker is not set. Set MAX_INFLIGHT_PER_WORKER or pass --max-inflight-per-worker.`, + ); + } + if (PIPELINED) { + MAX_INFLIGHT_PER_WORKER = + maxInflightPerWorker == 0 ? Infinity : maxInflightPerWorker!; + } console.log( - `[${connector.name}] max inflight per worker: ${MAX_INFLIGHT_PER_WORKER}`, + `[${connector.name}] pipelined=${PIPELINED} max-inflight-per-worker=${PIPELINED ? MAX_INFLIGHT_PER_WORKER : 'n/a'} pool-max=${runtimeConfig.poolMax}`, ); const run = async (seconds: number) => { const start = performance.now(); @@ -143,6 +147,41 @@ export async function runOne({ let completedWithinWindow = 0; let completedTotal = 0; + // === per-second time-series tracking === + const intervalMs = 1000; + const series: { + tSec: number; + tps: number; + p50_ms: number; + p95_ms: number; + p99_ms: number; + samples: number; + }[] = []; + const intervalHist = hdr.build({ + lowestDiscernibleValue: 1, + highestTrackableValue: 10_000_000_000, + numberOfSignificantValueDigits: 3, + }); + let intervalCount = 0; + + const intervalTimer = setInterval(() => { + const now = performance.now(); + // Stop recording once we've passed the test window + if (now > endAt) return; + const elapsedSec = (now - start) / 1000; + const samples = intervalCount; + series.push({ + tSec: Math.round(elapsedSec * 10) / 10, + tps: samples * (1000 / intervalMs), + p50_ms: samples ? intervalHist.getValueAtPercentile(50) / 1000 : 0, + p95_ms: samples ? intervalHist.getValueAtPercentile(95) / 1000 : 0, + p99_ms: samples ? intervalHist.getValueAtPercentile(99) / 1000 : 0, + samples, + }); + intervalCount = 0; + intervalHist.reset(); + }, intervalMs); + // Track when workers reach end of test window (before waiting for in-flight ops) let workersReachedEnd = 0; let resolveTestWindowEnd: () => void; @@ -223,7 +262,10 @@ export async function runOne({ completedTotal++; if (t1 <= endAt) { completedWithinWindow++; - hist.recordValue(Math.max(1, Math.round((t1 - t0) * 1e3))); + const latencyUs = Math.max(1, Math.round((t1 - t0) * 1e3)); + hist.recordValue(latencyUs); + intervalHist.recordValue(latencyUs); + intervalCount++; } } } @@ -254,7 +296,10 @@ export async function runOne({ completedTotal++; if (t1 <= endAt) { completedWithinWindow++; - hist.recordValue(Math.max(1, Math.round((t1 - t0) * 1e3))); + const latencyUs = Math.max(1, Math.round((t1 - t0) * 1e3)); + hist.recordValue(latencyUs); + intervalHist.recordValue(latencyUs); + intervalCount++; } } catch (err) { if (logErrors) { @@ -306,23 +351,29 @@ export async function runOne({ worker(i), ); - // Wait for all workers to reach end of test window (before they wait for in-flight ops) - await testWindowEndPromise; + try { + // Wait for all workers to reach end of test window (before they wait for in-flight ops) + await testWindowEndPromise; - const testWindowEndTime = performance.now(); - console.log( - `[${connector.name}] Test window ended at ${((testWindowEndTime - start) / 1000).toFixed(2)}s; waiting for in-flight operations...`, - ); + const testWindowEndTime = performance.now(); + console.log( + `[${connector.name}] Test window ended at ${((testWindowEndTime - start) / 1000).toFixed(2)}s; waiting for in-flight operations...`, + ); - // Now wait for all workers to fully complete (including in-flight ops) - await Promise.all(workerPromises); + // Now wait for all workers to fully complete (including in-flight ops) + await Promise.all(workerPromises); + } finally { + // Ensure the per-second sampler stops even if a worker throws. + clearInterval(intervalTimer); + } - return { start, completedWithinWindow, completedTotal }; + return { start, completedWithinWindow, completedTotal, series }; }; console.log(`[${connector.name}] Starting workers for ${seconds}s run...`); - const { start, completedWithinWindow, completedTotal } = await run(seconds); + const { start, completedWithinWindow, completedTotal, series } = + await run(seconds); console.log( `[${connector.name}] All workers finished (including in-flight ops)`, @@ -397,5 +448,6 @@ export async function runOne({ collision_ops: c.total, collision_count: c.collisions, collision_rate: c.collisionRate, + timeSeries: series, }; } diff --git a/templates/keynote-2/src/core/types.ts b/templates/keynote-2/src/core/types.ts index ca9815945cd..22a1fe632b7 100644 --- a/templates/keynote-2/src/core/types.ts +++ b/templates/keynote-2/src/core/types.ts @@ -1,3 +1,12 @@ +export type TimeSeriesPoint = { + tSec: number; + tps: number; + p50_ms: number; + p95_ms: number; + p99_ms: number; + samples: number; +}; + export type RunResult = { tps: number; samples: number; @@ -7,4 +16,5 @@ export type RunResult = { collision_ops: number; collision_count: number; collision_rate: number; + timeSeries: TimeSeriesPoint[]; }; diff --git a/templates/keynote-2/src/init/init_convex.ts b/templates/keynote-2/src/init/init_convex.ts index 5fcb630294c..1fae6e0c786 100644 --- a/templates/keynote-2/src/init/init_convex.ts +++ b/templates/keynote-2/src/init/init_convex.ts @@ -50,7 +50,7 @@ export async function initConvex(config: ConvexInitConfig) { } // Max ~16k writes per function; keep a safety margin - const CHUNK = 10_000; + const CHUNK = 7_000; console.log( `[convex] seeding ${accounts} accounts in chunks of ${CHUNK} (initial=${initialBalance})`, ); @@ -65,6 +65,7 @@ export async function initConvex(config: ConvexInitConfig) { count, initial: initialBalance, }); + await new Promise(r => setTimeout(r, 500)); } console.log('[convex] seed complete.'); diff --git a/templates/keynote-2/src/opts.ts b/templates/keynote-2/src/opts.ts index 98474c53082..cbac40e7d52 100644 --- a/templates/keynote-2/src/opts.ts +++ b/templates/keynote-2/src/opts.ts @@ -4,6 +4,7 @@ import { defaultBenchTestName, defaultDemoSystems, getSharedRuntimeDefaults, + parseAlphaList, parseStdbCompression, parseConnectorList, type BenchOptions, @@ -142,7 +143,11 @@ function addSharedRuntimeOptions(parser: CLIParser): CLIParser { return parser .option('--seconds ', 'Number of seconds to benchmark for', num()) .option('--concurrency ', 'Concurrent clients to run', num()) - .option('--alpha ', 'Alpha value', num()) + .option( + '--alpha ', + 'Zipf alpha. Accepts a single value or a comma-separated list (e.g. `0,1.5`).', + str(), + ) .option('--accounts ', 'Number of accounts to run with', num()) .option( '--initial-balance ', @@ -299,11 +304,16 @@ export function parseDemoOptions(argv: string[] = process.argv): DemoOptions { const runtimeOptions = resolveRuntimeOptions(options, runtimeDefaults); + const demoAlphas = parseAlphaList( + options.alpha as string | number | string[] | undefined, + '--alpha', + ); + return { ...runtimeOptions, - seconds: options.seconds ?? 10, - concurrency: options.concurrency ?? 50, - alpha: options.alpha ?? 1.5, + seconds: options.seconds ?? 300, + concurrency: options.concurrency ?? 64, + alpha: demoAlphas?.[0] ?? 1.5, systems: options.systems ?? options.connectors ?? [...defaultDemoSystems], skipPrep: options.skipPrep ?? false, @@ -329,6 +339,15 @@ export function parseBenchOptions(argv: string[] = process.argv): BenchOptions { type: 'strings', possibleValues: validConnectors, }) + .option( + '--runs ', + 'Repeat each (connector, alpha) combination this many times. One JSON written per run.', + num(), + ) + .option( + '--prep-between-alphas', + 'Run `pnpm run prep` before each (connector, alpha) combination to reset DB state.', + ) .option( '--contention-tests ', 'Run alpha sweep as start,end,step,concurrency', @@ -371,13 +390,20 @@ export function parseBenchOptions(argv: string[] = process.argv): BenchOptions { } : null; + const parsedAlphas = parseAlphaList( + options.alpha as string | number | string[] | undefined, + '--alpha', + ); + return { ...runtimeOptions, testName: args[0] ?? defaultBenchTestName, - seconds: options.seconds ?? 10, + seconds: options.seconds ?? 300, concurrency: - contentionTests?.concurrency ?? options.concurrency ?? 50, - alpha: concurrencyTests?.alpha ?? options.alpha ?? 1.5, + contentionTests?.concurrency ?? options.concurrency ?? 64, + alphas: parsedAlphas ?? [concurrencyTests?.alpha ?? 1.5], + runs: options.runs ?? 1, + prepBetweenAlphas: options.prepBetweenAlphas ?? false, connectors: options.connectors ?? options.systems ?? null, contentionTests, concurrencyTests, diff --git a/templates/keynote-2/src/rpc-servers/cockroach-rpc-server.ts b/templates/keynote-2/src/rpc-servers/cockroach-rpc-server.ts index 852d13c2e68..1ad9b8d3285 100644 --- a/templates/keynote-2/src/rpc-servers/cockroach-rpc-server.ts +++ b/templates/keynote-2/src/rpc-servers/cockroach-rpc-server.ts @@ -6,7 +6,7 @@ import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core'; import { eq, inArray, sql } from 'drizzle-orm'; import { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts'; import { getSharedRuntimeDefaults } from '../config.ts'; - +import { withTxnRetry } from './retry.ts'; const CRDB_URL = process.env.CRDB_URL; if (!CRDB_URL) { throw new Error('CRDB_URL not set'); @@ -43,36 +43,38 @@ async function rpcTransfer(args: Record) { const delta = BigInt(amount); - await db.transaction(async (tx) => { - const rows = await tx - .select() - .from(accounts) - .where(inArray(accounts.id, [fromId, toId])) - .for('update') - .orderBy(accounts.id); - - if (rows.length !== 2) { - throw new Error('account_missing'); - } + await withTxnRetry(() => + db.transaction(async (tx) => { + const rows = await tx + .select() + .from(accounts) + .where(inArray(accounts.id, [fromId, toId])) + .for('update') + .orderBy(accounts.id); + + if (rows.length !== 2) { + throw new Error('account_missing'); + } - const [first, second] = rows; - const fromRow = first.id === fromId ? first : second; - const toRow = first.id === fromId ? second : first; + const [first, second] = rows; + const fromRow = first.id === fromId ? first : second; + const toRow = first.id === fromId ? second : first; - if (fromRow.balance < delta) { - return; - } + if (fromRow.balance < delta) { + return; + } - await tx - .update(accounts) - .set({ balance: fromRow.balance - delta }) - .where(eq(accounts.id, fromId)); + await tx + .update(accounts) + .set({ balance: fromRow.balance - delta }) + .where(eq(accounts.id, fromId)); - await tx - .update(accounts) - .set({ balance: toRow.balance + delta }) - .where(eq(accounts.id, toId)); - }); + await tx + .update(accounts) + .set({ balance: toRow.balance + delta }) + .where(eq(accounts.id, toId)); + }), + ); } async function rpcGetAccount(args: Record) { diff --git a/templates/keynote-2/src/rpc-servers/postgres-rpc-server.ts b/templates/keynote-2/src/rpc-servers/postgres-rpc-server.ts index 272b4deff4d..1b52025b599 100644 --- a/templates/keynote-2/src/rpc-servers/postgres-rpc-server.ts +++ b/templates/keynote-2/src/rpc-servers/postgres-rpc-server.ts @@ -6,7 +6,7 @@ import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core'; import { eq, inArray, sql } from 'drizzle-orm'; import { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts'; import { getSharedRuntimeDefaults } from '../config.ts'; - +import { withTxnRetry } from './retry.ts'; const PG_URL = process.env.PG_URL; if (!PG_URL) { throw new Error('PG_URL not set'); @@ -43,36 +43,38 @@ async function rpcTransfer(args: Record) { const delta = BigInt(amount); - await db.transaction(async (tx) => { - const rows = await tx - .select() - .from(accounts) - .where(inArray(accounts.id, [fromId, toId])) - .for('update') - .orderBy(accounts.id); - - if (rows.length !== 2) { - throw new Error('account_missing'); - } + await withTxnRetry(() => + db.transaction(async (tx) => { + const rows = await tx + .select() + .from(accounts) + .where(inArray(accounts.id, [fromId, toId])) + .for('update') + .orderBy(accounts.id); + + if (rows.length !== 2) { + throw new Error('account_missing'); + } - const [first, second] = rows; - const fromRow = first.id === fromId ? first : second; - const toRow = first.id === fromId ? second : first; + const [first, second] = rows; + const fromRow = first.id === fromId ? first : second; + const toRow = first.id === fromId ? second : first; - if (fromRow.balance < delta) { - return; - } + if (fromRow.balance < delta) { + return; + } - await tx - .update(accounts) - .set({ balance: fromRow.balance - delta }) - .where(eq(accounts.id, fromId)); + await tx + .update(accounts) + .set({ balance: fromRow.balance - delta }) + .where(eq(accounts.id, fromId)); - await tx - .update(accounts) - .set({ balance: toRow.balance + delta }) - .where(eq(accounts.id, toId)); - }); + await tx + .update(accounts) + .set({ balance: toRow.balance + delta }) + .where(eq(accounts.id, toId)); + }), + ); } async function rpcGetAccount(args: Record) { diff --git a/templates/keynote-2/src/rpc-servers/retry.ts b/templates/keynote-2/src/rpc-servers/retry.ts new file mode 100644 index 00000000000..d048f8a3ca4 --- /dev/null +++ b/templates/keynote-2/src/rpc-servers/retry.ts @@ -0,0 +1,61 @@ +// Retry serialization (40001), deadlock (40P01), lock-not-available (55P03). +const RETRYABLE_SQLSTATES = new Set(['40001', '40P01', '55P03']); + +export type RetryOptions = { + maxAttempts?: number; + baseDelayMs?: number; + maxDelayMs?: number; + onRetry?: (attempt: number, err: unknown) => void; +}; + +export class RetryExhaustedError extends Error { + constructor( + public readonly attempts: number, + public readonly lastError: unknown, + ) { + const cause = + (lastError as { message?: string })?.message ?? String(lastError); + super(`retry exhausted after ${attempts} attempts: ${cause}`); + this.name = 'RetryExhaustedError'; + } +} + +function getSqlstate(err: unknown): string | undefined { + let cursor: any = err; + for (let i = 0; i < 5 && cursor; i++) { + const code = cursor.code; + if (typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code)) return code; + cursor = cursor.cause ?? cursor.originalError ?? cursor.innerError ?? null; + } + return undefined; +} + +function isRetryable(err: unknown): boolean { + const code = getSqlstate(err); + return code !== undefined && RETRYABLE_SQLSTATES.has(code); +} + +export async function withTxnRetry( + fn: () => Promise, + options: RetryOptions = {}, +): Promise { + const maxAttempts = options.maxAttempts ?? 10; + const baseDelayMs = options.baseDelayMs ?? 5; + const maxDelayMs = options.maxDelayMs ?? 200; + + let lastError: unknown; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await fn(); + } catch (err) { + lastError = err; + if (!isRetryable(err)) throw err; + if (attempt === maxAttempts) throw new RetryExhaustedError(attempt, err); + options.onRetry?.(attempt, err); + const cap = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt); + const delay = Math.floor(Math.random() * cap); + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + } + } + throw new RetryExhaustedError(maxAttempts, lastError); +} diff --git a/templates/keynote-2/src/rpc-servers/supabase-rpc-server.ts b/templates/keynote-2/src/rpc-servers/supabase-rpc-server.ts index 81c0565d821..d90d27139ad 100644 --- a/templates/keynote-2/src/rpc-servers/supabase-rpc-server.ts +++ b/templates/keynote-2/src/rpc-servers/supabase-rpc-server.ts @@ -6,7 +6,7 @@ import { pgTable, integer, bigint as pgBigint } from 'drizzle-orm/pg-core'; import { eq, inArray, sql } from 'drizzle-orm'; import type { RpcRequest, RpcResponse } from '../connectors/rpc/rpc_common.ts'; import { getSharedRuntimeDefaults } from '../config.ts'; - +import { withTxnRetry } from './retry.ts'; const DB_URL = process.env.SUPABASE_DB_URL ?? process.env.PG_URL; if (!DB_URL) { throw new Error('SUPABASE_DB_URL / PG_URL not set'); @@ -43,37 +43,39 @@ async function rpcTransfer(args: Record) { const delta = BigInt(amount); - await db.transaction(async (tx) => { - const rows = await tx - .select() - .from(accounts) - .where(inArray(accounts.id, [fromId, toId])) - .for('update') - .orderBy(accounts.id); - - if (rows.length !== 2) { - throw new Error('account_missing'); - } + await withTxnRetry(() => + db.transaction(async (tx) => { + const rows = await tx + .select() + .from(accounts) + .where(inArray(accounts.id, [fromId, toId])) + .for('update') + .orderBy(accounts.id); + + if (rows.length !== 2) { + throw new Error('account_missing'); + } - const [first, second] = rows; - const fromRow = first.id === fromId ? first : second; - const toRow = first.id === fromId ? second : first; + const [first, second] = rows; + const fromRow = first.id === fromId ? first : second; + const toRow = first.id === fromId ? second : first; - if (fromRow.balance < delta) { - // Not enough balance; just skip - return; - } + if (fromRow.balance < delta) { + // Not enough balance; just skip + return; + } - await tx - .update(accounts) - .set({ balance: fromRow.balance - delta }) - .where(eq(accounts.id, fromId)); + await tx + .update(accounts) + .set({ balance: fromRow.balance - delta }) + .where(eq(accounts.id, fromId)); - await tx - .update(accounts) - .set({ balance: toRow.balance + delta }) - .where(eq(accounts.id, toId)); - }); + await tx + .update(accounts) + .set({ balance: toRow.balance + delta }) + .where(eq(accounts.id, toId)); + }), + ); } async function rpcGetAccount(args: Record) { diff --git a/templates/keynote-2/src/scenario_recipes/rpc_single_call.ts b/templates/keynote-2/src/scenario_recipes/rpc_single_call.ts index 796b633bd2c..c7cf6738545 100644 --- a/templates/keynote-2/src/scenario_recipes/rpc_single_call.ts +++ b/templates/keynote-2/src/scenario_recipes/rpc_single_call.ts @@ -20,16 +20,5 @@ export async function rpc_single_call( : 'transfer:transfer' : 'transfer'; - for (let attempts = 0; attempts < 3; attempts++) { - try { - await api.call(fn, { amount, from_id: from, to_id: to }); - return; - } catch (e: any) { - const msg = String(e?.message ?? ''); - const retriable = /429|502|503|504/.test(msg); - if (!retriable || attempts === 2) throw e; - - await new Promise((r) => setTimeout(r, 50 * (attempts + 1))); - } - } + await api.call(fn, { amount, from_id: from, to_id: to }); }