From 8bcc659c788da9828163683281e83ddba0dc3414 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 04:09:39 +0000 Subject: [PATCH 1/3] docs(benchmarks-website): fold planning/ into code-adjacent docs Replace the nine-file `benchmarks-website/planning/` directory with project-level READMEs and module-level `//!` docs anchored to the code they describe. - New `benchmarks-website/README.md` is the v3 entry point: side-by-side with v2 explanation, architecture summary, route table, local-dev and deployment recipes, and the cutover plan. - New `benchmarks-website/AGENTS.md` carries just the still-load-bearing agent norms (don't touch v2, don't add a server-side classifier, the tooltip / slider / wheel-pan footguns). - `server/src/lib.rs` gains a crate-level module map + request-flow walkthrough (was: components/server.md). - `server/src/schema.rs` gains the full design-principles + per-table contract walkthrough (was: 01-schema.md). - `server/src/records.rs` and `server/src/ingest.rs` gain the wire envelope + HTTP-matrix docs (was: 02-contracts.md). - `vortex-bench/src/v3.rs` gains the producer mapping + per-binary inventory + per-suite query dim values table (was: benchmark-mapping.md and components/emitter.md). - The TODO breadcrumb on `api::charts::collect_group_charts` is rewritten to match the way the user already tracks the N+1 follow-up; the removed `planning/README.md` cross-reference goes away. - Decisions from `decisions.md` are folded into the relevant module doc where they still drive current behavior; everything else (alpha history, deferred phase-2 items) is dropped. Signed-off-by: Claude --- benchmarks-website/AGENTS.md | 76 ++++++ benchmarks-website/README.md | 108 +++++++++ benchmarks-website/planning/00-overview.md | 104 -------- benchmarks-website/planning/01-schema.md | 228 ------------------ benchmarks-website/planning/02-contracts.md | 227 ----------------- benchmarks-website/planning/AGENTS.md | 172 ------------- benchmarks-website/planning/README.md | 188 --------------- .../planning/benchmark-mapping.md | 147 ----------- .../planning/components/emitter.md | 86 ------- .../planning/components/server.md | 70 ------ .../planning/components/web-ui.md | 62 ----- benchmarks-website/planning/decisions.md | 95 -------- benchmarks-website/planning/deferred.md | 118 --------- benchmarks-website/server/src/api/charts.rs | 9 +- benchmarks-website/server/src/api/dto.rs | 8 +- benchmarks-website/server/src/api/mod.rs | 7 +- benchmarks-website/server/src/ingest.rs | 30 ++- benchmarks-website/server/src/lib.rs | 63 ++++- benchmarks-website/server/src/records.rs | 44 +++- benchmarks-website/server/src/schema.rs | 100 +++++++- vortex-bench/src/datasets/mod.rs | 7 +- vortex-bench/src/v3.rs | 86 ++++++- 22 files changed, 501 insertions(+), 1534 deletions(-) create mode 100644 benchmarks-website/AGENTS.md create mode 100644 benchmarks-website/README.md delete mode 100644 benchmarks-website/planning/00-overview.md delete mode 100644 benchmarks-website/planning/01-schema.md delete mode 100644 benchmarks-website/planning/02-contracts.md delete mode 100644 benchmarks-website/planning/AGENTS.md delete mode 100644 benchmarks-website/planning/README.md delete mode 100644 benchmarks-website/planning/benchmark-mapping.md delete mode 100644 benchmarks-website/planning/components/emitter.md delete mode 100644 benchmarks-website/planning/components/server.md delete mode 100644 benchmarks-website/planning/components/web-ui.md delete mode 100644 benchmarks-website/planning/decisions.md delete mode 100644 benchmarks-website/planning/deferred.md diff --git a/benchmarks-website/AGENTS.md b/benchmarks-website/AGENTS.md new file mode 100644 index 00000000000..bf00e48b855 --- /dev/null +++ b/benchmarks-website/AGENTS.md @@ -0,0 +1,76 @@ + + +# AGENTS.md — `benchmarks-website/` + +Read [`README.md`](README.md) first for the architecture and the v2/v3 +side-by-side situation. Then this file. The root [`CLAUDE.md`](../CLAUDE.md) +covers Rust style, test layout, commit conventions. + +## Don't touch the v2 site + +Until the cutover PR lands, the top-level v2 files +(`server.js`, `src/`, `index.html`, `vite.config.js`, `package.json`, +`package-lock.json`, `public/`, the top-level `Dockerfile`, +`docker-compose.yml`, `ec2-init.txt`) and the `benchmarks-website` service +in `docker-compose.yml` and the `publish-benchmarks-website.yml` workflow +are production. Don't edit them as part of unrelated work. + +## v3 specifics + +- **Wire shapes are a coordinated change.** [`server/src/records.rs`](server/src/records.rs), + [`vortex-bench/src/v3.rs`](../vortex-bench/src/v3.rs), and (until cutover) + [`migrate/src/classifier.rs`](migrate/src/classifier.rs) must agree. + Bumping a shape means changing all three plus the snapshot fixtures in + one commit. +- **`measurement_id` is server-internal.** Never put it on the wire. It is + a deterministic hash over `commit_sha` plus the dim tuple, computed in + [`server/src/db.rs`](server/src/db.rs) and reused by the migrator via + the same crate. +- **Don't write a server-side classifier for live ingest.** The emitter + produces v3-shape records directly; the migrator's classifier only + exists to translate v2 records once and goes away after cutover. +- **Don't reach for WASM.** SSR + a thin hydration script in + [`server/static/chart-init.js`](server/static/chart-init.js) is the + whole client. +- **Don't re-introduce a server-side commit cap.** `?n=all` is the default + for HTML routes; visual downsampling happens client-side via LTTB on the + visible commit range only. +- **Don't refetch on every scope change.** The chart fetches its full + history once. Pan, zoom, slider, and the range strip rebuild in place + via the in-memory LTTB pass on the cached payload. The single exception + is the inline-payload zoom-out path: when the user zooms past the first + group's inlined `LANDING_INLINE_N` window for the first time, + `chart-init.js` lazy-fetches `?n=all` once and replaces the payload. + +## Footguns we have already hit + +- **Reverse predecessor walk in the tooltip.** `payload.commits[]` is + sorted oldest-first by SQL — `commits[0]` is the oldest, `commits[N-1]` + is the newest. For per-row delta the predecessor of `commits[idx]` is + at `idx - 1`. We caught a regression where a "fix" flipped this to + `idx + 1`; the original walk-backward direction is right. +- **`pointer-events: auto` on the tooltip host.** The tooltip is + positioned at the cursor; making it pointer-interactive causes a + flicker loop. Keep it `pointer-events: none` and offset via + `transform: translate(12px, 12px)`. +- **`change` events on the slider.** Use `input` events with a small + throttle; `change` only fires on release and feels broken. + +## Local dev + +```bash +INGEST_BEARER_TOKEN=dev cargo run -p vortex-bench-server +cargo nextest run -p vortex-bench-server -p vortex-bench-migrate +INSTA_UPDATE=auto cargo nextest run -p vortex-bench-server # update snapshots +``` + +For the migrator end-to-end against the real S3 dump: + +```bash +cargo run -p vortex-bench-migrate -- run --output ./bench.duckdb +VORTEX_BENCH_DB=./bench.duckdb INGEST_BEARER_TOKEN=dev \ + cargo run -p vortex-bench-server +``` diff --git a/benchmarks-website/README.md b/benchmarks-website/README.md new file mode 100644 index 00000000000..6460b6c07b3 --- /dev/null +++ b/benchmarks-website/README.md @@ -0,0 +1,108 @@ + + +# bench.vortex.dev + +The website behind `bench.vortex.dev`. The directory currently houses **two +implementations side by side**, run together until the v3 cutover lands: + +- **v2** (top-level files: `server.js`, `src/`, `index.html`, `vite.config.js`, + `package.json`, `Dockerfile`, `docker-compose.yml`, `ec2-init.txt`, + `public/`). The Node + React stack that has shipped to production for the + life of the site. Built and published by + [`.github/workflows/publish-benchmarks-website.yml`](../.github/workflows/publish-benchmarks-website.yml). +- **v3** (`server/` + `migrate/`). A single Rust binary — + [`vortex-bench-server`](server/) — that owns a DuckDB file on local disk, + serves the API, and renders the HTML. Compiles all static assets + (`chart.umd.js`, `chart-init.js`, `style.css`) into the binary so deploys + are one file plus a database. Container image at + `ghcr.io/vortex-data/vortex/vortex-bench-server:latest`. + [`migrate/`](migrate/) is a one-shot tool that loads v2's S3 dataset into a + v3 DuckDB; it is throwaway and goes away after cutover. + +Live results are produced by +[`.github/workflows/bench.yml`](../.github/workflows/bench.yml) and +[`.github/workflows/sql-benchmarks.yml`](../.github/workflows/sql-benchmarks.yml), +which CI runs after every push to `develop`. Until cutover the same payload is +emitted to both stacks (v2 via the legacy `--gh-json` path appended to a public +S3 bucket; v3 via `--gh-json-v3` POSTed to `/api/ingest`). + +## v3 architecture in one paragraph + +`axum` (HTTP) + `maud` (compile-time HTML) + embedded `duckdb-rs` over a single +local DB file. Five fact tables (`query_measurements`, `compression_times`, +`compression_sizes`, `random_access_times`, `vector_search_runs`) plus a +`commits` dim table — see [`server/src/schema.rs`](server/src/schema.rs) for +the column contracts. Three HTML routes (`/`, `/chart/{slug}`, +`/group/{slug}`) and four JSON routes (`GET /api/groups`, +`GET /api/chart/{slug}`, `GET /api/group/{slug}`, `GET /health`), plus a +bearer-gated `POST /api/ingest`. Charts render inline on the landing page via +SSR + lazy hydration; visual downsampling (LTTB at most +`MAX_VISIBLE_POINTS = 500`) is client-side in +[`server/static/chart-init.js`](server/static/chart-init.js). + +For the per-module crate map and the request-flow walkthrough, see the +`//!` doc on [`server/src/lib.rs`](server/src/lib.rs). The producer side of +the ingest contract lives in +[`vortex-bench/src/v3.rs`](../vortex-bench/src/v3.rs); the historical-data +side in [`migrate/src/classifier.rs`](migrate/src/classifier.rs). + +## Local dev + +```bash +# v3 server (DuckDB lives at ./bench.duckdb by default). +INGEST_BEARER_TOKEN=dev cargo run -p vortex-bench-server +# server logs: "bench server listening addr=127.0.0.1:3000 db=bench.duckdb" + +# v3 historical migrator (writes a fully populated DuckDB the server can open). +cargo run -p vortex-bench-migrate -- run --output ./bench.duckdb +``` + +Ingest fixture data via the snapshot tests' envelopes (see +[`server/tests/common/mod.rs`](server/tests/common/mod.rs)) or by hand-rolling +a JSONL file and POSTing through `scripts/post-ingest.py`. + +```bash +cargo nextest run -p vortex-bench-server -p vortex-bench-migrate +INSTA_UPDATE=auto cargo nextest run -p vortex-bench-server # update snapshots +``` + +For the v2 stack: + +```bash +cd benchmarks-website +npm install +npm run dev +``` + +## Deployment + +`docker-compose.yml` runs both stacks side by side: v2 on `:80` and v3 on +`:3001`. `watchtower` polls GHCR every 60s so a fresh image push lands +automatically. v3 reads `INGEST_BEARER_TOKEN` from +`/etc/vortex-bench/secrets.env`, persists DuckDB to +`/opt/benchmarks-website/data/bench.duckdb`, and binds `0.0.0.0:3000` so the +container's `:3001` host port forwards through. + +The v3 server is throwaway-friendly: every request runs against the local +DuckDB file, and a fresh boot reapplies the schema DDL idempotently. The +migrator deletes the target file (and its `.wal`) before populating it, so +re-running `vortex-bench-migrate run --output ...` is safe. + +## Cutover plan (in flight) + +The work to flip `bench.vortex.dev` from v2 to v3 is tracked outside this +repo. The relevant code-side bits: + +- v3 deploys today on a separate EC2 host and is exercised by CI's dual-write + step against a test bearer token. +- v2 keeps shipping unchanged until DNS flips. **Do not touch the top-level + v2 files unless you are doing the cleanup PR opened post-flip.** +- The v2 cleanup PR removes everything top-level under `benchmarks-website/` + that belongs to v2 (`server.js`, `src/`, `index.html`, `vite.config.js`, + `package.json`, `package-lock.json`, `public/`, the top-level `Dockerfile`, + `docker-compose.yml`, `ec2-init.txt`, and the + `publish-benchmarks-website.yml` workflow). The v3 tree under `server/` and + `migrate/` is untouched. diff --git a/benchmarks-website/planning/00-overview.md b/benchmarks-website/planning/00-overview.md deleted file mode 100644 index c6c3e05c57f..00000000000 --- a/benchmarks-website/planning/00-overview.md +++ /dev/null @@ -1,104 +0,0 @@ - - -# 00 - Overview - -## What we're building - -A replacement for the current `bench.vortex.dev` site. The new -stack is a **single Rust binary** (axum + maud + duckdb-rs) that -owns a **DuckDB database** on local disk and serves the website -plus an `/api/ingest` route. CI eventually POSTs new benchmark -results there. There is no separate ingester service, no S3 -coordination layer for writes, no client-side WASM. - -The server crate is `vortex-bench-server` at -`benchmarks-website/server/`. - -## Phasing - -We build this in two phases. **Plan only the first.** - -### Alpha (this plan) - -The smallest end-to-end loop that proves the design: - -1. **Schema** locked enough to ingest one benchmark result. -2. **Server**: open DuckDB, accept a bearer-token-authenticated POST, - serve a couple of read routes. -3. **Emitter**: `vortex-bench --gh-json-v3` + a tiny POST script. -4. **Web UI**: one landing page + one chart page rendered against a - fixture DB. - -That's it. No production deploy, no historical data import, no CI -workflow integration, no admin tooling, no schema migration -framework, no auth beyond the shared bearer token. All of those -live in [`deferred.md`](./deferred.md). - -The alpha runs on a developer machine. v2 keeps running in -production unchanged. There is no cutover in alpha. - -### Phase 2 and beyond - -Once the alpha loop is green, we layer in production deploy, -historical migration, CI dual-write, and the rest of the v2-parity -work. Stubs are in [`deferred.md`](./deferred.md). - -## Architecture (alpha) - -One process, one DB file. The server is the API and the website. -The emitter writes JSONL of bare records; a small POST script -wraps and uploads them. CI isn't wired up yet; ingest happens -manually during alpha. - -## Components - -Three components for alpha. Each is one workstream, one branch, one -PR. - -| Component | Plan | Owns | -|---|---|---| -| Server | [components/server.md](./components/server.md) | DuckDB open + schema, bearer-auth ingest, read routes, HTML routes mounted from web-ui | -| Emitter | [components/emitter.md](./components/emitter.md) | `vortex-bench --gh-json-v3` + the post-ingest script | -| Web UI | [components/web-ui.md](./components/web-ui.md) | Landing page + chart page, against a fixture DuckDB | - -### Dependencies - -The schema feeds all three components. The contracts feed the -server and the emitter. With both stable, **all three components -can be worked on in parallel**. - -## Goals - -In priority order: - -1. **End-to-end alpha loop works.** Emit → POST → store → render. -2. **Schema is the right shape.** Five fact tables (one per - measurement family) plus a `commits` dim. See - [`01-schema.md`](./01-schema.md). -3. **Each component is small enough that one agent can finish it - in one PR.** No mega-PRs. - -Cutover, parity, and "faster than v2" are explicit non-goals at -alpha; they come back in phase 2. - -## Shared docs - -- [`00-overview.md`](./00-overview.md) (this file) -- [`01-schema.md`](./01-schema.md) - the five fact tables + `commits` -- [`02-contracts.md`](./02-contracts.md) - wire shapes + HTTP error - matrix + auth header -- [`benchmark-mapping.md`](./benchmark-mapping.md) - existing - benchmarks → fact tables -- [`decisions.md`](./decisions.md) - resolved decisions -- [`deferred.md`](./deferred.md) - phase-2 stubs - -## Status of v2 during alpha - -v2 stays in production untouched. Do not edit -`benchmarks-website/server.js`, `benchmarks-website/src/`, or any -other v2 files at `benchmarks-website/` top level. v3 lives in the -sibling subdirectory at `benchmarks-website/server/` -(`vortex-bench-server` crate). diff --git a/benchmarks-website/planning/01-schema.md b/benchmarks-website/planning/01-schema.md deleted file mode 100644 index dfc6b05ba27..00000000000 --- a/benchmarks-website/planning/01-schema.md +++ /dev/null @@ -1,228 +0,0 @@ - - -# 01 - DuckDB schema (alpha) - -The persistent data model. **One `commits` dim table plus five fact -tables, one per measurement family.** No lookup tables, no views, no -migration framework; those are deferred (see -[`deferred.md`](./deferred.md)). - -## Design principles - -1. **One fact table per (dim shape, value shape).** A row in any - fact table has every value column populated; NULLs only appear - in genuinely optional dimensions. -2. **No discriminator columns spanning families.** No `metric_kind` - enum forcing five shapes into one row. -3. **No JSON escape hatch.** New benchmark parameters become real - columns. Adding a nullable column is cheap; the readability win - is worth it. -4. **Hashed primary key per table.** Each fact table has a - `measurement_id` that is a deterministic 64-bit hash of - `commit_sha` plus that table's dimensional tuple. Including - `commit_sha` makes every (commit, dim) pair a distinct row - - that's what the chart pages render as a time series. - Server-internal; not on the wire. -5. **`commits` is the only dim table.** Engine, format, dataset, - etc. stay as inline strings; DuckDB's dictionary encoding makes - a lookup table pointless. -6. **Ratios are not stored.** Computed at query time from - `compression_sizes`. - -## Why five fact tables, not one - -The five families have genuinely different shapes: - -| Table | Shape sketch | -|---|---| -| `query_measurements` | dataset + query_idx + engine + format + storage → timing **and** memory | -| `compression_times` | dataset + format + op∈{encode,decode} → timing | -| `compression_sizes` | dataset + format → bytes | -| `random_access_times` | dataset + format → timing (different dataset namespace) | -| `vector_search_runs` | dataset + layout + flavor + threshold → timing + counters | - -Forcing them into one table either bloats every row with columns -that are NULL for ~99% of rows (`layout`, `flavor`, `threshold`, -`matches`, `rows_scanned`, `bytes_scanned`) or splits scan results -across multiple rows that have to be re-joined to render one chart. - -## Group / chart / series fit - -The render-time view used by `/api/groups` and `/api/chart/:slug` -is mechanically derivable per table: - -| Table | Group key | Chart key | Series key | -|---|---|---|---| -| `query_measurements` | `(dataset, dataset_variant, scale_factor, storage)` | `(dataset, query_idx)` | `(engine, format)` | -| `compression_times` | constant `"Compression"` | `(dataset, dataset_variant)` | `(format, op)` | -| `compression_sizes` | constant `"Compression Size"` | `(dataset, dataset_variant)` | `format` | -| `random_access_times` | constant `"Random Access"` | `dataset` | `format` | -| `vector_search_runs` | `(dataset, layout)` | `(dataset, layout, threshold)` | `flavor` | - -The classifier logic in v2's `v2-classifier.js` mostly disappears - -each table already knows what suite it represents. - -## Tables - -DDL is the server's call. Below is the column contract: name, type -family, and whether it's NOT NULL. The server agent picks exact -DuckDB types, indexes, and constraint syntax. - -### `commits` (dim) - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `commit_sha` | string | yes (PK) | 40-hex lowercase | -| `timestamp` | timestamptz | yes | | -| `message` | string | optional | first line only | -| `author_name` | string | optional | | -| `author_email` | string | optional | | -| `committer_name` | string | optional | | -| `committer_email` | string | optional | | -| `tree_sha` | string | yes | | -| `url` | string | yes | | - -Populated from the envelope on every `/api/ingest` call. - -### `query_measurements` - -SQL query suites: TPC-H, TPC-DS, ClickBench, StatPopGen, -PolarSignals, Fineweb, GhArchive, Public-BI. Memory columns are -populated when the run was instrumented for memory; NULL otherwise. -Timing and memory share the row because they're produced together -for the same query execution. - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `measurement_id` | int64 | yes (PK) | hash of dim tuple | -| `commit_sha` | string | yes | FK to `commits` | -| `dataset` | string | yes | `tpch`, `tpcds`, `clickbench`, ... | -| `dataset_variant` | string | optional | ClickBench flavor, Public-BI name | -| `scale_factor` | string | optional | TPC SF; n_rows for StatPopGen / PolarSignals | -| `query_idx` | int32 | yes | 1-based | -| `storage` | string | yes | `nvme` or `s3` | -| `engine` | string | yes | `datafusion`, `duckdb`, `vortex`, `arrow` | -| `format` | string | yes | `vortex-file-compressed`, `parquet`, `lance`, ... | -| `value_ns` | int64 | yes | median timing, ns | -| `all_runtimes_ns` | list<int64> | yes | per-iteration timings | -| `peak_physical` | int64 | optional | bytes | -| `peak_virtual` | int64 | optional | bytes | -| `physical_delta` | int64 | optional | bytes | -| `virtual_delta` | int64 | optional | bytes | -| `env_triple` | string | optional | e.g. `x86_64-linux-gnu` | - -### `compression_times` - -Encode/decode timings from `compress-bench`. - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `measurement_id` | int64 | yes (PK) | | -| `commit_sha` | string | yes | FK | -| `dataset` | string | yes | | -| `dataset_variant` | string | optional | | -| `format` | string | yes | | -| `op` | string | yes | `encode` or `decode` | -| `value_ns` | int64 | yes | | -| `all_runtimes_ns` | list<int64> | yes | | -| `env_triple` | string | optional | | - -### `compression_sizes` - -On-disk sizes from `compress-bench`. One-shot, no per-iteration data. -Compression ratios in v2 (`vortex:parquet-zstd ratio/...`) are a -SELECT over this table joined to itself; they're not stored. - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `measurement_id` | int64 | yes (PK) | | -| `commit_sha` | string | yes | FK | -| `dataset` | string | yes | | -| `dataset_variant` | string | optional | | -| `format` | string | yes | | -| `value_bytes` | int64 | yes | | - -### `random_access_times` - -Take-time timings from `random-access-bench`. Different dataset -namespace from `compression_times` - kept in its own table so -dataset filters never have to disambiguate which suite a row -belongs to. - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `measurement_id` | int64 | yes (PK) | | -| `commit_sha` | string | yes | FK | -| `dataset` | string | yes | | -| `format` | string | yes | | -| `value_ns` | int64 | yes | | -| `all_runtimes_ns` | list<int64> | yes | | -| `env_triple` | string | optional | | - -### `vector_search_runs` - -Cosine-similarity scans from `vector-search-bench`. The only family -that emits a timing **plus side counters** for the same scan; -keeping them in one row avoids a 1:N split that has to be re-joined -on read. - -| Column | Type | Required? | Notes | -|---|---|---|---| -| `measurement_id` | int64 | yes (PK) | | -| `commit_sha` | string | yes | FK | -| `dataset` | string | yes | e.g. `cohere-large-10m` | -| `layout` | string | yes | `TrainLayout`, e.g. `partitioned` | -| `flavor` | string | yes | `VectorFlavor`, e.g. `vortex-turboquant` | -| `threshold` | double | yes | cosine threshold | -| `value_ns` | int64 | yes | per-scan wall time | -| `all_runtimes_ns` | list<int64> | yes | | -| `matches` | int64 | yes | | -| `rows_scanned` | int64 | yes | | -| `bytes_scanned` | int64 | yes | | -| `iterations` | int32 | yes | not part of the dim hash | -| `env_triple` | string | optional | | - -## `measurement_id` hash - -Per-table xxhash64 over `commit_sha` plus that table's dimensional -tuple. Including `commit_sha` makes every (commit, dim) pair a -distinct row, which is what the chart pages render as a time -series. The hash is **server-internal** - the wire never carries -it. The server's INSERT path computes it before each -`INSERT ... ON CONFLICT DO UPDATE`, which gives idempotent upsert -on re-emission of the same (commit, dim) pair. Encoding details -(input order, NULL handling, byte layout) are the server's call, -since the value never crosses a process boundary. - -When the historical migrator lands (deferred), it reuses the -server's hash function via a shared crate. - -## Storage values - -`storage` is `'nvme'` or `'s3'`. Legacy `gcs` is dropped. Only -`query_measurements` carries `storage` - the other families don't -fan out by storage backend. - -## Schema changes during alpha - -There is no migration framework. If you change the schema: - -1. Update this doc. -2. Update the server's DDL. -3. Delete any local `bench.duckdb` and re-run. - -A real forward-only migration framework lands post-alpha. See -[`deferred.md`](./deferred.md). - -## What's intentionally NOT here (deferred) - -- `schema_meta` and migration framework. -- `known_engines` / `known_formats` / `known_datasets` lookup - tables and seed SQL. -- Views (`v_compression_ratios`, `v_latest_per_group`, etc.). -- Pre-downsampled aliases. -- A `microbench_runs` table - reserved as the next family to add - when microbench results start landing. diff --git a/benchmarks-website/planning/02-contracts.md b/benchmarks-website/planning/02-contracts.md deleted file mode 100644 index 8f32072123d..00000000000 --- a/benchmarks-website/planning/02-contracts.md +++ /dev/null @@ -1,227 +0,0 @@ - - -# 02 - Wire contracts (alpha) - -The cross-component glue between the emitter, the POST script, and -the server. Wire-format only - implementations are local to each -component. - -If two components disagree about a shape, **this file is right** -and both update. - -## Records are discriminated by `kind` - -Each record on the wire carries a `kind` field that picks one of -the [five fact tables](./01-schema.md#tables). The emitter never -decides "what column" - it decides "what kind", and the rest of the -row is that kind's flat field set. - -| `kind` | Destination table | -|---|---| -| `query_measurement` | `query_measurements` | -| `compression_time` | `compression_times` | -| `compression_size` | `compression_sizes` | -| `random_access_time` | `random_access_times` | -| `vector_search_run` | `vector_search_runs` | - -**Unknown `kind` values cause a 400.** Unknown fields within a known -`kind` also cause a 400. Version skew should fail loudly. - -## Per-kind record shapes - -All shared metadata first; per-kind fields after. - -### `query_measurement` - -| Field | Type | Required? | Notes | -|---|---|---|---| -| `kind` | `"query_measurement"` | yes | discriminator | -| `commit_sha` | string | yes | 40-hex lowercase | -| `dataset` | string | yes | `tpch`, `tpcds`, `clickbench`, ... | -| `dataset_variant` | string | optional | ClickBench flavor, Public-BI name | -| `scale_factor` | string | optional | TPC SF; n_rows for StatPopGen / PolarSignals | -| `query_idx` | integer | yes | 1-based | -| `storage` | enum string | yes | `nvme` or `s3` | -| `engine` | string | yes | `datafusion`, `duckdb`, `vortex`, `arrow` | -| `format` | string | yes | `vortex-file-compressed`, `parquet`, `lance`, ... | -| `value_ns` | integer | yes | median timing, ns | -| `all_runtimes_ns` | array<integer> | yes | per-iteration timings (may be empty) | -| `peak_physical` | integer | optional | bytes | -| `peak_virtual` | integer | optional | bytes | -| `physical_delta` | integer | optional | bytes | -| `virtual_delta` | integer | optional | bytes | -| `env_triple` | string | optional | e.g. `x86_64-linux-gnu` | - -The four memory fields are populated together (all four or none). - -### `compression_time` - -| Field | Type | Required? | Notes | -|---|---|---|---| -| `kind` | `"compression_time"` | yes | | -| `commit_sha` | string | yes | | -| `dataset` | string | yes | | -| `dataset_variant` | string | optional | | -| `format` | string | yes | | -| `op` | enum string | yes | `encode` or `decode` | -| `value_ns` | integer | yes | | -| `all_runtimes_ns` | array<integer> | yes | | -| `env_triple` | string | optional | | - -### `compression_size` - -| Field | Type | Required? | Notes | -|---|---|---|---| -| `kind` | `"compression_size"` | yes | | -| `commit_sha` | string | yes | | -| `dataset` | string | yes | | -| `dataset_variant` | string | optional | | -| `format` | string | yes | | -| `value_bytes` | integer | yes | | - -### `random_access_time` - -| Field | Type | Required? | Notes | -|---|---|---|---| -| `kind` | `"random_access_time"` | yes | | -| `commit_sha` | string | yes | | -| `dataset` | string | yes | random-access dataset name (e.g. `chimp`, `taxi`) | -| `format` | string | yes | | -| `value_ns` | integer | yes | | -| `all_runtimes_ns` | array<integer> | yes | | -| `env_triple` | string | optional | | - -### `vector_search_run` - -| Field | Type | Required? | Notes | -|---|---|---|---| -| `kind` | `"vector_search_run"` | yes | | -| `commit_sha` | string | yes | | -| `dataset` | string | yes | e.g. `cohere-large-10m` | -| `layout` | string | yes | `TrainLayout`, e.g. `partitioned` | -| `flavor` | string | yes | `VectorFlavor`, e.g. `vortex-turboquant` | -| `threshold` | number | yes | cosine threshold | -| `value_ns` | integer | yes | per-scan wall time (median of iterations) | -| `all_runtimes_ns` | array<integer> | yes | | -| `matches` | integer | yes | | -| `rows_scanned` | integer | yes | | -| `bytes_scanned` | integer | yes | | -| `iterations` | integer | yes | | -| `env_triple` | string | optional | | - -## Ingest envelope - -`/api/ingest` accepts one envelope per POST. The envelope wraps a -heterogeneous batch of records (any mix of `kind`s). Required -top-level fields: - -- `run_meta`: object with `benchmark_id` (string), `schema_version` - (integer; `1` at alpha), `started_at` (RFC 3339 timestamp). -- `commit`: object with the columns of the [`commits` - table](./01-schema.md#commits-dim), keyed by their column names - with `commit_sha` renamed to `sha`. The server upserts this row - before applying records. -- `records`: array of per-`kind` records as defined above. - -`vortex-bench --gh-json-v3 ` writes JSONL of bare records -only. The envelope (`run_meta` + `commit`) is added by the -post-ingest script before POSTing - this keeps the Rust emitter -dependency-light. - -The post-ingest script is responsible for filling the `commit` -fields. CI has the SHA from `${{ github.sha }}`; the rest comes -from `git show` or equivalent. See -[`components/emitter.md`](./components/emitter.md). - -## HTTP matrix for `POST /api/ingest` - -| Condition | Status | -|---|---| -| Happy path | 200 with `{ "inserted": N, "updated": M }` | -| Malformed JSON | 400 | -| Unknown `kind`, unknown field, or per-record validation failure | 400 with the offending record index | -| Missing/invalid bearer token | 401 | -| Schema version newer than server expects | 409 | -| Other server error | 500 | - -All-or-nothing per POST: a single failed record fails the whole -batch. The reported `inserted` and `updated` counts are aggregated -across all five tables. - -## Authentication header - -```text -Authorization: Bearer -``` - -Compared with constant-time equality on the server. Token comes from -the `INGEST_BEARER_TOKEN` env var. - -## Slug grammar (server ↔ web-ui) - -The web-ui receives slugs from `/api/groups` and feeds them back -into `/api/chart/:slug`. Slugs are **opaque strings** as far as the -web-ui is concerned: it never parses or constructs them itself, -only echoes what the API returned. The server is free to choose any -slug format, change it without breaking the web-ui, or make it -debuggable (e.g. `qm-tpch-q01-nvme-sf1`) - the only contract is -"`/api/chart/:slug` accepts any slug `/api/groups` returned." - -## Read API - -Four JSON routes today. Field shapes are not binding; refine during -implementation. - -### `GET /api/groups` - -A flat list of distinct group keys derivable from the data, with -just enough metadata to link to a chart. The server walks each fact -table to produce the group keys defined in -[`01-schema.md`](./01-schema.md#group--chart--series-fit). Every -chart entry includes a `slug` that round-trips through -`/api/chart/:slug`, and every group has its own `slug` that -round-trips through `/api/group/:slug`. - -### `GET /api/chart/:slug` - -Returns the data for one chart: a `display_name`, a `unit_kind`, an -ordered `commits` list (sha + timestamp + first-line message + url), -and a `series` map keyed by series name where each value is an -array aligned to `commits` (with `null` for missing data points). -Accepts `?n=&y=&mode=&hidden=` to scope the commit window and -configure the rendered view. - -`unit_kind` is a small structured taxonomy that tells the client -*what* the values are. Wire values stay in the kind's base unit; the -client picks a display unit (e.g. `ms` for `time_ns` values around -1e6) so the rendered axis stays readable. Worked example: -`12,000,000,000` ns on the wire → `12 s` on the y-axis. - -| `unit_kind` | Base unit on the wire | Client display picker | -|---------------------|-------------------------|-------------------------------| -| `time_ns` | nanoseconds | `ns | µs | ms | s` by magnitude | -| `bytes` | bytes | `B | KiB | MiB | GiB | TiB` (binary) | -| `ratio` | dimensionless ratio | identity (no suffix) | -| `count` | dimensionless count | identity (no suffix) | -| `throughput_mb_s` | megabytes per second | identity, `MB/s` suffix | - -Adding a variant is a wire-compat change: bump the emitter, the -migrator, and the client unit picker in `chart-init.js` together. - -### `GET /api/group/:slug` - -Returns every chart in a group as a single batch payload, in render -order. Used by the `/group/{slug}` HTML page and (today) by the -landing page hydration path. Same query parameters as -`/api/chart/:slug`. - -### `GET /health` - -Returns `{ status, db_path, schema_version, latest_commit_timestamp, -row_counts }`. Cheap; suitable for load-balancer health checks. - -Per-commit page, range queries, and the rest of the read API are -deferred. See [`deferred.md`](./deferred.md). diff --git a/benchmarks-website/planning/AGENTS.md b/benchmarks-website/planning/AGENTS.md deleted file mode 100644 index 719c47b4c22..00000000000 --- a/benchmarks-website/planning/AGENTS.md +++ /dev/null @@ -1,172 +0,0 @@ - - -# AGENTS.md - benchmarks-website v3 - -Brief for coding agents working on the v3 rewrite of `bench.vortex.dev`. Keep this file short. -Detail belongs in component plans. - -## Status - -Alpha is shipped. The v3 server, migrator, and inline-charts UI are all merged to -`ct/benchmarks-v3`. The current focus is **production readiness**: secrets, CI ingestion wiring, -smoke-testing on a real host, the DNS flip, and v2 cleanup. See [`README.md`](./README.md) for the -live punch list. - -The v2 site (top-level files in `benchmarks-website/`: `server.js`, `src/`, `package.json`, -`index.html`, `Dockerfile`, `docker-compose.yml`, `ec2-init.txt`, etc.) is still in production on -`bench.vortex.dev` and **stays running unchanged** until the DNS flip. The v3 server lives alongside -it as `vortex-bench-server` at `benchmarks-website/server/`. - -## Architecture in 10 bullets - -- Single Rust binary: `axum` (HTTP) + `maud` (SSR HTML) + embedded `duckdb-rs`. All static assets - (`chart.umd.js`, `chart-init.js`, `style.css`) are `include_bytes!`'d into the binary. No CDN. - A `tower-http` `CompressionLayer` wraps every response (gzip/brotli). -- One DuckDB file on local disk holds five fact tables (compression time, query measurement, vector - search, RAG, random access) plus a `commits` dim table. Schema in - [`01-schema.md`](./01-schema.md). -- One ingest endpoint: `POST /api/ingest`, gated by a static bearer token from the - `INGEST_BEARER_TOKEN` env var. Wire shapes in [`02-contracts.md`](./02-contracts.md). -- Three HTML routes — `/`, `/chart/{slug}`, `/group/{slug}` — and four JSON routes — - `GET /api/groups`, `GET /api/chart/{slug}`, `GET /api/group/{slug}`, `GET /health` — all served - from the same binary. -- `ChartKey` and `GroupKey` enums round-trip through URLs as `.` - slugs. No DB lookup required to decode a URL. -- Charts render inline on the landing page. Each `` is paired with a - `` JSON out of an /// HTML body. Returns `None` if the script tag isn't present. -pub fn extract_chart_data(body: &str, idx: usize) -> Option { +pub(crate) fn extract_chart_data(body: &str, idx: usize) -> Option { let needle = format!(r#"")? + start; @@ -337,7 +337,7 @@ pub fn extract_chart_data(body: &str, idx: usize) -> Option { /// Configure `insta` to look for snapshots in `tests/snapshots/` keyed by /// just the explicit name (no module prefix). Every test in this crate uses /// these settings so the snapshot file layout is path-independent. -pub fn insta_settings() -> insta::Settings { +pub(crate) fn insta_settings() -> insta::Settings { let mut s = insta::Settings::clone_current(); s.set_snapshot_path("snapshots"); s.set_prepend_module_to_snapshot(false); @@ -347,7 +347,10 @@ pub fn insta_settings() -> insta::Settings { /// Lift a single chart slug from `/api/groups`, picking from a group whose /// name matches `predicate`. Used by tests that need a real slug to drive /// `/chart/{slug}` and `/api/chart/{slug}` round-trips. -pub async fn pick_chart_slug(server: &Server, predicate: impl Fn(&str) -> bool) -> Result { +pub(crate) async fn pick_chart_slug( + server: &Server, + predicate: impl Fn(&str) -> bool, +) -> Result { let client = reqwest::Client::new(); let groups: Value = client .get(server.url("/api/groups")) @@ -369,7 +372,10 @@ pub async fn pick_chart_slug(server: &Server, predicate: impl Fn(&str) -> bool) /// Lift a single group slug from `/api/groups`, picking the first group /// whose name matches `predicate`. -pub async fn pick_group_slug(server: &Server, predicate: impl Fn(&str) -> bool) -> Result { +pub(crate) async fn pick_group_slug( + server: &Server, + predicate: impl Fn(&str) -> bool, +) -> Result { let client = reqwest::Client::new(); let groups: Value = client .get(server.url("/api/groups")) @@ -389,7 +395,7 @@ pub async fn pick_group_slug(server: &Server, predicate: impl Fn(&str) -> bool) /// Look up a group entry by its `name` field inside an `/api/groups` /// response. -pub fn group_by_name<'a>(groups: &'a Value, name: &str) -> Result<&'a Value> { +pub(crate) fn group_by_name<'a>(groups: &'a Value, name: &str) -> Result<&'a Value> { groups["groups"] .as_array() .context("groups is array")? @@ -400,7 +406,7 @@ pub fn group_by_name<'a>(groups: &'a Value, name: &str) -> Result<&'a Value> { /// Fuzzy `f64` equality for test assertions. The summary rollups round-trip /// through SQL so exact equality isn't safe even on integer-valued inputs. -pub fn assert_close(actual: f64, expected: f64) { +pub(crate) fn assert_close(actual: f64, expected: f64) { let delta = (actual - expected).abs(); assert!( delta < 0.000_001, @@ -412,7 +418,7 @@ pub fn assert_close(actual: f64, expected: f64) { /// filter dropdown — its trigger button and the chip panel. Keeps the /// snapshot focused on the chip markup and stable against changes elsewhere /// on the page. -pub fn filter_bar_section(body: &str) -> String { +pub(crate) fn filter_bar_section(body: &str) -> String { let needle = r#"
".to_string(); @@ -447,7 +453,7 @@ pub fn filter_bar_section(body: &str) -> String { /// Pull the `
` containing chips for one /// dimension (`"engine"` or `"format"`). -pub fn filter_section(body: &str, dim: &str) -> String { +pub(crate) fn filter_section(body: &str, dim: &str) -> String { let bar = filter_bar_section(body); let needle = format!(r#"data-filter="{dim}""#); let Some(_) = bar.find(&needle) else { @@ -464,7 +470,7 @@ pub fn filter_section(body: &str, dim: &str) -> String { } /// Pull a single chip's opening tag for assertions. -pub fn extract_chip(section: &str, value: &str) -> String { +pub(crate) fn extract_chip(section: &str, value: &str) -> String { let needle = format!(r#"data-value="{value}""#); let Some(idx) = section.find(&needle) else { return String::new(); diff --git a/benchmarks-website/server/tests/ingest.rs b/benchmarks-website/server/tests/ingest.rs index a2cf46f8632..96018bfa417 100644 --- a/benchmarks-website/server/tests/ingest.rs +++ b/benchmarks-website/server/tests/ingest.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Integration tests covering the acceptance criteria from -//! `benchmarks-website/planning/components/server.md`. +//! Integration tests for `POST /api/ingest` — round-trips the bearer +//! check, the all-or-nothing transaction, the schema-version gate, and +//! the upsert path. use std::net::SocketAddr; From b7d80456ca0a68adbe963113a4b8f1243144f4d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 6 May 2026 15:01:00 +0000 Subject: [PATCH 3/3] docs(benchmarks-website): address review feedback on planning-docs migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc-accuracy nits surfaced in review of #7810. No behavior changes. - `vortex-bench/src/datasets/mod.rs::Dataset::v3_dataset_dims`: drop the stale "Public-BI emits dataset = public-bi, dataset_variant = …" example. The only override (`PBIBenchmark::v3_dataset_dims`) actually returns `(&self.name, None)` to match the migrate classifier — the parent-namespace framing was historical. Reword to "Override only when a suite needs a different dataset name on the wire than its `name()` returns." - `query_idx` field doc on both `vortex-bench/src/v3.rs` and `server/src/records.rs`: drop the "1-based" claim. Query indices are whatever the producing bench loop happens to use (ClickBench is 0-based, others vary); both sides agree only because the migrate classifier parses literal digits out of `q07`-style v2 chart names. - `server/src/app.rs`: add `/api/group/{slug}` to the read-API line; the router wires it but the module doc had drifted. - `server/src/api/charts.rs::collect_group_charts`: re-introduce a `TODO(#7812):` marker so the N+1 follow-up is grep-discoverable. Tracking issue: https://github.com/vortex-data/vortex/issues/7812 - `benchmarks-website/README.md`: rewrite the cutover-plan bullet to reflect that v3 runs alongside v2 on the same EC2 host (v2 on `:80`, v3 on `:3001`), per `docker-compose.yml` and `ec2-init.txt`. The earlier "separate EC2 host" claim was stale. - `server/src/ingest.rs`: add the missing 400 row for the older-than- expected `schema_version` case to the HTTP matrix table. - `vortex-bench/src/v3.rs`: drop the `[`canonical_tpc_scale_factor`]` intra-doc link in the new dim-mapping table — the function is private, so `RUSTDOCFLAGS="-D warnings" cargo doc` rejected it. - `server/src/lib.rs`: rename "PNGs" to "logos" in the route table — typos parsed "PN" inside "PNGs" as a typo of "ON" and failed `Spell Check with Typos`. Verification: - `cargo +nightly fmt --all` - `cargo build -p vortex-bench-server -p vortex-bench-migrate` → clean. - `cargo test -p vortex-bench-server -p vortex-bench-migrate` → all 140+ tests pass. - `cargo clippy --all-targets --all-features -p vortex-bench-server -p vortex-bench-migrate -- -W dead_code -W unreachable_pub -W unused_imports` → 0 warnings. - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps -p vortex-bench -p vortex-bench-server -p vortex-bench-migrate` → clean. - `typos .` → 0 hits. Signed-off-by: Claude --- benchmarks-website/README.md | 4 ++-- benchmarks-website/server/src/api/charts.rs | 7 ++++--- benchmarks-website/server/src/app.rs | 3 ++- benchmarks-website/server/src/ingest.rs | 3 ++- benchmarks-website/server/src/lib.rs | 2 +- benchmarks-website/server/src/records.rs | 5 ++++- vortex-bench/src/datasets/mod.rs | 7 +++---- vortex-bench/src/v3.rs | 7 +++++-- 8 files changed, 23 insertions(+), 15 deletions(-) diff --git a/benchmarks-website/README.md b/benchmarks-website/README.md index 6460b6c07b3..63604539279 100644 --- a/benchmarks-website/README.md +++ b/benchmarks-website/README.md @@ -96,8 +96,8 @@ re-running `vortex-bench-migrate run --output ...` is safe. The work to flip `bench.vortex.dev` from v2 to v3 is tracked outside this repo. The relevant code-side bits: -- v3 deploys today on a separate EC2 host and is exercised by CI's dual-write - step against a test bearer token. +- v3 runs alongside v2 on the same EC2 host today (v2 on `:80`, v3 on + `:3001`) and is fed by CI's dual-write `--gh-json-v3` path. - v2 keeps shipping unchanged until DNS flips. **Do not touch the top-level v2 files unless you are doing the cleanup PR opened post-flip.** - The v2 cleanup PR removes everything top-level under `benchmarks-website/` diff --git a/benchmarks-website/server/src/api/charts.rs b/benchmarks-website/server/src/api/charts.rs index b0edfb0380c..28184279067 100644 --- a/benchmarks-website/server/src/api/charts.rs +++ b/benchmarks-website/server/src/api/charts.rs @@ -75,10 +75,11 @@ pub(crate) fn chart_payload( /// Collect every chart inside one group. Returns `None` if the group has no /// data at all (callers should render a 404). /// -/// Known N+1: this re-runs the entire [`collect_groups`] discovery pass per -/// call before fetching each chart, so the landing page is +/// TODO(#7812): this re-runs the entire [`collect_groups`] discovery pass +/// per call before fetching each chart, so the landing page is /// O(groups * charts_per_group) DB queries plus the discovery scan. Fine -/// for the current dataset; the rewrite is owned outside this branch. +/// for the current dataset; tracked for the refactor that collapses it +/// into a single query. pub(crate) fn collect_group_charts( conn: &Connection, key: &GroupKey, diff --git a/benchmarks-website/server/src/app.rs b/benchmarks-website/server/src/app.rs index 759205431b1..d013bfe9ad7 100644 --- a/benchmarks-website/server/src/app.rs +++ b/benchmarks-website/server/src/app.rs @@ -4,7 +4,8 @@ //! Axum [`Router`] composition and shared [`AppState`]. //! //! The router mounts: -//! - `/api/groups`, `/api/chart/{slug}`, `/health` (read API) +//! - `/api/groups`, `/api/chart/{slug}`, `/api/group/{slug}`, `/health` +//! (read API) //! - `/api/ingest` (gated by [`crate::auth::require_bearer`]) //! - HTML routes contributed by [`crate::html::router`] //! diff --git a/benchmarks-website/server/src/ingest.rs b/benchmarks-website/server/src/ingest.rs index 9b6844dca44..b97401bea8a 100644 --- a/benchmarks-website/server/src/ingest.rs +++ b/benchmarks-website/server/src/ingest.rs @@ -15,7 +15,8 @@ //! | Malformed JSON or unknown field at the envelope level | 400 | //! | Unknown `kind`, unknown record field, or per-record validation fail | 400 with the offending record's index | //! | Missing or invalid bearer token | 401 (raised by [`crate::auth::require_bearer`]) | -//! | Schema version newer than this server expects | 409 | +//! | Schema version newer than this server expects | 409 | +//! | Schema version older than this server expects | 400 (via the malformed-envelope path) | //! | Other server error | 500 | //! //! All-or-nothing semantics: a single failed record fails the whole batch diff --git a/benchmarks-website/server/src/lib.rs b/benchmarks-website/server/src/lib.rs index 9794f887476..3174aa50696 100644 --- a/benchmarks-website/server/src/lib.rs +++ b/benchmarks-website/server/src/lib.rs @@ -20,7 +20,7 @@ //! HTML; the rest are shells fetched on first toggle. //! - `GET /chart/{slug}` — single-chart permalink. //! - `GET /group/{slug}` — every chart in one group on a single page. -//! - `GET /static/...` — the bundled JS / CSS / PNGs. +//! - `GET /static/...` — the bundled JS / CSS / logos. //! - `GET /api/groups` — flat list of every group with chart-link metadata. //! - `GET /api/chart/{slug}` — one chart's payload (`commits`, `series`, //! `unit_kind`, ...). diff --git a/benchmarks-website/server/src/records.rs b/benchmarks-website/server/src/records.rs index e71128b759e..446b701e296 100644 --- a/benchmarks-website/server/src/records.rs +++ b/benchmarks-website/server/src/records.rs @@ -134,7 +134,10 @@ pub struct QueryMeasurement { /// TPC SF as a string. Populated for TPC-H/TPC-DS, NULL elsewhere. #[serde(default)] pub scale_factor: Option, - /// 1-based query index inside the suite. + /// Query index within the suite. The convention (0-based or 1-based) is + /// fixed per suite by the producing bench loop; the migrate classifier + /// matches it by parsing literal digits out of `q07`-style v2 chart + /// names. pub query_idx: i32, /// Storage backend the run targeted: `nvme` or `s3`. Validated on insert. pub storage: String, diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs index d89f99d36f4..d35d3f869e0 100644 --- a/vortex-bench/src/datasets/mod.rs +++ b/vortex-bench/src/datasets/mod.rs @@ -40,10 +40,9 @@ pub trait Dataset { /// Map this dataset to the v3 `(dataset, dataset_variant)` pair emitted /// in `compression_*` records. /// - /// Default: `(name(), None)`. Override for suites that have a parent - /// namespace and a sub-dataset (e.g. Public-BI emits - /// `dataset = "public-bi"`, `dataset_variant = ""`). - /// The query-side equivalent is documented on + /// Default: `(name(), None)`. Override only when a suite needs a + /// different dataset name on the wire than its `name()` returns. The + /// query-side equivalent is documented on /// [`crate::v3::benchmark_dataset_dims`]. fn v3_dataset_dims(&self) -> (&str, Option<&str>) { (self.name(), None) diff --git a/vortex-bench/src/v3.rs b/vortex-bench/src/v3.rs index c2dabca1ff9..99c85314fbe 100644 --- a/vortex-bench/src/v3.rs +++ b/vortex-bench/src/v3.rs @@ -134,7 +134,10 @@ pub struct QueryMeasurementRecord { /// a per-suite scale factor. #[serde(skip_serializing_if = "Option::is_none")] pub scale_factor: Option, - /// 1-based query index within the suite. + /// Query index within the suite. The convention (0-based or 1-based) is + /// fixed per suite by the producing bench loop; the migrate classifier + /// matches it by parsing literal digits out of `q07`-style v2 chart + /// names. pub query_idx: u32, /// Storage backend the run targeted (`nvme` or `s3`). pub storage: String, @@ -282,7 +285,7 @@ fn canonical_tpc_scale_factor(scale_factor: &str) -> String { /// /// | `BenchmarkDataset` | `dataset` | `dataset_variant` | `scale_factor` | Notes | /// |---|---|---|---|---| -/// | `TpcH { scale_factor }` | `tpch` | `None` | TPC SF as string (`"1"`, `"10"`, `"100"`, `"1000"`) | Run through [`canonical_tpc_scale_factor`] so `"1.0"` and `"1"` collapse. | +/// | `TpcH { scale_factor }` | `tpch` | `None` | TPC SF as string (`"1"`, `"10"`, `"100"`, `"1000"`) | Run through `canonical_tpc_scale_factor` so `"1.0"` and `"1"` collapse. | /// | `TpcDS { scale_factor }` | `tpcds` | `None` | TPC SF as string | Same canonicalization as TPC-H. | /// | `ClickBench { flavor: _ }` | `clickbench` | `None` | `None` | Migrate path drops flavor; live emitter matches so historical and live merge. | /// | `StatPopGen { n_rows: _ }` | `statpopgen` | `None` | `None` | Migrate path carries no SF for this suite; live drops it for the same reason. |