diff --git a/.gitignore b/.gitignore index 51e30794ec..db5e288297 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ arcticdb/store # Python venvs created by install scripts myenv + +# Pivot's local datastores and generated server configs +pivot-parquet/catalog +pivot-parquet/config.yaml +pivot-parquet-partitioned/catalog +pivot-parquet-partitioned/config.yaml diff --git a/pivot-parquet-partitioned/benchmark.sh b/pivot-parquet-partitioned/benchmark.sh new file mode 100755 index 0000000000..1251353052 --- /dev/null +++ b/pivot-parquet-partitioned/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" +exec ../lib/benchmark-common.sh diff --git a/pivot-parquet-partitioned/check b/pivot-parquet-partitioned/check new file mode 100755 index 0000000000..617138474f --- /dev/null +++ b/pivot-parquet-partitioned/check @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +: "${PIVOT_PORT:=5432}" + +psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -c 'SELECT 1' >/dev/null diff --git a/pivot-parquet-partitioned/create.sql b/pivot-parquet-partitioned/create.sql new file mode 100644 index 0000000000..e492f36d7e --- /dev/null +++ b/pivot-parquet-partitioned/create.sql @@ -0,0 +1,117 @@ +-- Pivot-flavoured ClickBench schema. Maps the original ClickBench schema onto +-- the SQL types pivot's planner understands (BIGINT, INTEGER, SMALLINT, +-- VARCHAR). EventTime/EventDate are stored in the source parquet as packed +-- epoch seconds / days, so they're declared as their integer storage type here; +-- the queries read them as real dates on demand via +-- `make_timestamp(EventTime * 1000000)` (the function counts microseconds, as it +-- does in DuckDB, while the column counts seconds) and `make_date(EventDate)`. CHAR(...) columns are widened to VARCHAR. +-- +-- `load` substitutes `{source}` with the directory holding hits.parquet. +DROP TABLE IF EXISTS hits; +CREATE TABLE hits ( + WatchID BIGINT, + JavaEnable SMALLINT, + Title VARCHAR, + GoodEvent SMALLINT, + EventTime BIGINT, + EventDate USMALLINT, + CounterID INTEGER, + ClientIP INTEGER, + RegionID INTEGER, + UserID BIGINT, + CounterClass SMALLINT, + OS SMALLINT, + UserAgent SMALLINT, + URL VARCHAR, + Referer VARCHAR, + IsRefresh SMALLINT, + RefererCategoryID SMALLINT, + RefererRegionID INTEGER, + URLCategoryID SMALLINT, + URLRegionID INTEGER, + ResolutionWidth SMALLINT, + ResolutionHeight SMALLINT, + ResolutionDepth SMALLINT, + FlashMajor SMALLINT, + FlashMinor SMALLINT, + FlashMinor2 VARCHAR, + NetMajor SMALLINT, + NetMinor SMALLINT, + UserAgentMajor SMALLINT, + UserAgentMinor VARCHAR, + CookieEnable SMALLINT, + JavascriptEnable SMALLINT, + IsMobile SMALLINT, + MobilePhone SMALLINT, + MobilePhoneModel VARCHAR, + Params VARCHAR, + IPNetworkID INTEGER, + TraficSourceID SMALLINT, + SearchEngineID SMALLINT, + SearchPhrase VARCHAR, + AdvEngineID SMALLINT, + IsArtifical SMALLINT, + WindowClientWidth SMALLINT, + WindowClientHeight SMALLINT, + ClientTimeZone SMALLINT, + ClientEventTime BIGINT, + SilverlightVersion1 SMALLINT, + SilverlightVersion2 SMALLINT, + SilverlightVersion3 INTEGER, + SilverlightVersion4 SMALLINT, + PageCharset VARCHAR, + CodeVersion INTEGER, + IsLink SMALLINT, + IsDownload SMALLINT, + IsNotBounce SMALLINT, + FUniqID BIGINT, + OriginalURL VARCHAR, + HID INTEGER, + IsOldCounter SMALLINT, + IsEvent SMALLINT, + IsParameter SMALLINT, + DontCountHits SMALLINT, + WithHash SMALLINT, + HitColor VARCHAR, + LocalEventTime BIGINT, + Age SMALLINT, + Sex SMALLINT, + Income SMALLINT, + Interests SMALLINT, + Robotness SMALLINT, + RemoteIP INTEGER, + WindowName INTEGER, + OpenerName INTEGER, + HistoryLength SMALLINT, + BrowserLanguage VARCHAR, + BrowserCountry VARCHAR, + SocialNetwork VARCHAR, + SocialAction VARCHAR, + HTTPError SMALLINT, + SendTiming INTEGER, + DNSTiming INTEGER, + ConnectTiming INTEGER, + ResponseStartTiming INTEGER, + ResponseEndTiming INTEGER, + FetchTiming INTEGER, + SocialSourceNetworkID SMALLINT, + SocialSourcePage VARCHAR, + ParamPrice BIGINT, + ParamOrderID VARCHAR, + ParamCurrency VARCHAR, + ParamCurrencyID SMALLINT, + OpenstatServiceName VARCHAR, + OpenstatCampaignID VARCHAR, + OpenstatAdID VARCHAR, + OpenstatSourceID VARCHAR, + UTMSource VARCHAR, + UTMMedium VARCHAR, + UTMCampaign VARCHAR, + UTMContent VARCHAR, + UTMTerm VARCHAR, + FromTag VARCHAR, + HasGCLID SMALLINT, + RefererHash BIGINT, + URLHash BIGINT, + CLID INTEGER +) WITH (with_pre_existing_parquets = '{source}'); diff --git a/pivot-parquet-partitioned/data-size b/pivot-parquet-partitioned/data-size new file mode 100755 index 0000000000..7d804d3ff9 --- /dev/null +++ b/pivot-parquet-partitioned/data-size @@ -0,0 +1,5 @@ +#!/bin/bash +set -eu + +# No ingestion — the reported size is the parquet files the table reads. +du -bc data/hits_*.parquet | tail -n1 | cut -f1 diff --git a/pivot-parquet-partitioned/install b/pivot-parquet-partitioned/install new file mode 100755 index 0000000000..72f68986a0 --- /dev/null +++ b/pivot-parquet-partitioned/install @@ -0,0 +1,9 @@ +#!/bin/bash +set -eu + +sudo apt-get update -y +sudo apt-get install -y curl postgresql-client + +# The official installer: it puts the `pivot` binary under $HOME/.pivot (or +# $PIVOT_HOME) and needs no root. ./start runs it from there. +curl https://pivotlake.io | sh diff --git a/pivot-parquet-partitioned/load b/pivot-parquet-partitioned/load new file mode 100755 index 0000000000..be7ce0c815 --- /dev/null +++ b/pivot-parquet-partitioned/load @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu +: "${PIVOT_PORT:=5432}" + +# No ingestion: the table adopts the parquet files in place, so the files the +# driver downloaded have to stay around for the whole run. +mkdir -p data +mv hits_*.parquet data/ + +sed "s|{source}|$PWD/data|g" create.sql | + psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -v ON_ERROR_STOP=1 -f - + +sync diff --git a/pivot-parquet-partitioned/make-json.sh b/pivot-parquet-partitioned/make-json.sh new file mode 100755 index 0000000000..c1f862c352 --- /dev/null +++ b/pivot-parquet-partitioned/make-json.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Turns a finished run into results//.json. +# +# ./benchmark.sh +# ./make-json.sh c6a.4xlarge +# +# The per-try timings come from result.csv, which the driver writes as +# ,,; the metadata comes from template.json. The table +# adopts the downloaded parquet files in place, so the load is a catalog write and +# the data size is the files' own. +set -eu + +machine=${1:?usage: $0 } +date=$(date -u +%Y-%m-%d) +output="results/${date//-/}/$machine.json" + +mkdir -p "$(dirname "$output")" +jq -n \ + --slurpfile template template.json \ + --arg date "$date" \ + --arg machine "$machine" \ + --rawfile csv result.csv \ + '$template[0] + { + date: $date, + machine: $machine, + cluster_size: 1, + load_time: 0, + data_size: 14737666736, + result: ($csv | split("\n") | map(select(length > 0) | split(",")) + | group_by(.[0] | tonumber) | sort_by(.[0][0] | tonumber) + | map(map(.[2] | if . == "null" then null else tonumber end))) + }' > "$output" +echo "$output" diff --git a/pivot-parquet-partitioned/queries.sql b/pivot-parquet-partitioned/queries.sql new file mode 100644 index 0000000000..dbaebeb5b6 --- /dev/null +++ b/pivot-parquet-partitioned/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0 +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits +SELECT AVG(UserID) FROM hits +SELECT COUNT(DISTINCT UserID) FROM hits +SELECT COUNT(DISTINCT SearchPhrase) FROM hits +SELECT MIN(EventDate), MAX(EventDate) FROM hits +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10 +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10 +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10 +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10 +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10 +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10 +SELECT UserID, extract(minute FROM make_timestamp(EventTime * 1000000)) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID FROM hits WHERE UserID = 435090932899640449 +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%' +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10 +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25 +SELECT REGEXP_JIT_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25 +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10 +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10 +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10 +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10 +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10 +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000 +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000 +SELECT URLHash, make_date(EventDate) AS EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, make_date(EventDate) ORDER BY PageViews DESC LIMIT 10 OFFSET 100 +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000 +SELECT DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-14' AND make_date(EventDate) <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) ORDER BY DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) LIMIT 10 OFFSET 1000 diff --git a/pivot-parquet-partitioned/query b/pivot-parquet-partitioned/query new file mode 100755 index 0000000000..01f4f6279c --- /dev/null +++ b/pivot-parquet-partitioned/query @@ -0,0 +1,27 @@ +#!/bin/bash +# Reads a SQL query from stdin and runs it via psql against Pivot. +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line (parsed from +# psql's `\timing` "Time: ms" output). +# Exit non-zero on error. +set -e +: "${PIVOT_PORT:=5432}" + +query=$(cat) + +raw=$(psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -v ON_ERROR_STOP=1 -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? + +if [ "$exit_code" -ne 0 ] || printf '%s\n' "$raw" | grep -qE '^(ERROR|FATAL|PANIC):|psql: error'; then + printf '%s\n' "$raw" >&2 + exit 1 +fi + +printf '%s\n' "$raw" | grep -v '^Time:' + +ms=$(printf '%s\n' "$raw" | grep -oP 'Time:\s*\K[0-9.]+' | tail -n1) +if [ -z "$ms" ]; then + echo "no Time: in psql output" >&2 + exit 1 +fi + +awk -v m="$ms" 'BEGIN { printf "%.3f\n", m / 1000 }' >&2 diff --git a/pivot-parquet-partitioned/results/20261001/c6a.2xlarge.json b/pivot-parquet-partitioned/results/20261001/c6a.2xlarge.json new file mode 100644 index 0000000000..4167af9ab3 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.006, 0, 0], + [0.035, 0.01, 0.011], + [0.16, 0.039, 0.04], + [0.695, 0.043, 0.044], + [0.943, 0.301, 0.3], + [0.642, 0.467, 0.461], + [0.053, 0.026, 0.027], + [0.037, 0.013, 0.013], + [0.561, 0.446, 0.447], + [1.23, 0.701, 0.695], + [0.238, 0.098, 0.1], + [0.388, 0.132, 0.132], + [0.538, 0.183, 0.185], + [1.896, 0.52, 0.522], + [0.623, 0.212, 0.209], + [0.387, 0.26, 0.258], + [1.627, 0.688, 0.689], + [1.599, 0.564, 0.559], + [3.798, 1.482, 1.487], + [0.376, 0.011, 0.01], + [9.291, 0.466, 0.536], + [10.649, 0.587, 0.593], + [21.026, 12.382, 10.633], + [1.605, 0.062, 0.065], + [0.053, 0.014, 0.014], + [0.519, 0.064, 0.063], + [0.072, 0.014, 0.015], + [9.246, 0.597, 0.67], + [7.867, 2.136, 2.13], + [0.084, 0.03, 0.03], + [1.601, 0.252, 0.244], + [4.978, 0.309, 0.31], + [4.041, 1.052, 1.059], + [9.279, 8.436, 7.662], + [9.294, 8.265, 7.503], + [0.316, 0.271, 0.271], + [0.093, 0.021, 0.019], + [0.032, 0.011, 0.012], + [0.059, 0.01, 0.01], + [0.116, 0.04, 0.039], + [0.027, 0.009, 0.009], + [0.019, 0.01, 0.007], + [0.022, 0.012, 0.011] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c6a.4xlarge.json b/pivot-parquet-partitioned/results/20261001/c6a.4xlarge.json new file mode 100644 index 0000000000..b17a462bd2 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.006, 0, 0], + [0.025, 0.009, 0.009], + [0.049, 0.022, 0.022], + [0.089, 0.026, 0.025], + [0.21, 0.172, 0.172], + [0.582, 0.267, 0.268], + [0.025, 0.016, 0.014], + [0.027, 0.01, 0.011], + [0.457, 0.274, 0.274], + [0.746, 0.449, 0.451], + [0.115, 0.061, 0.063], + [0.144, 0.079, 0.081], + [0.529, 0.165, 0.163], + [1.792, 0.409, 0.408], + [0.592, 0.196, 0.194], + [0.187, 0.15, 0.15], + [1.582, 0.368, 0.363], + [1.586, 0.344, 0.343], + [3.644, 1.084, 1.07], + [0.073, 0.011, 0.011], + [9.185, 0.271, 0.271], + [10.778, 0.328, 0.322], + [21.027, 0.755, 0.8], + [1.603, 0.069, 0.07], + [0.04, 0.014, 0.013], + [0.481, 0.038, 0.038], + [0.043, 0.018, 0.015], + [9.285, 0.249, 0.247], + [7.804, 1.087, 1.074], + [0.047, 0.017, 0.018], + [1.49, 0.176, 0.177], + [4.972, 0.284, 0.284], + [4.36, 1.279, 1.277], + [9.181, 0.846, 0.85], + [9.031, 0.849, 0.845], + [0.185, 0.161, 0.158], + [0.041, 0.019, 0.02], + [0.024, 0.011, 0.011], + [0.047, 0.009, 0.01], + [0.078, 0.04, 0.044], + [0.02, 0.01, 0.008], + [0.019, 0.009, 0.009], + [0.018, 0.008, 0.008] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c6a.metal.json b/pivot-parquet-partitioned/results/20261001/c6a.metal.json new file mode 100644 index 0000000000..ea7d25f496 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.011, 0.002, 0.002], + [0.016, 0.005, 0.004], + [0.042, 0.01, 0.006], + [0.177, 0.007, 0.006], + [0.165, 0.043, 0.041], + [0.484, 0.053, 0.053], + [0.013, 0.005, 0.008], + [0.02, 0.01, 0.007], + [0.32, 0.055, 0.053], + [0.536, 0.096, 0.092], + [0.18, 0.021, 0.025], + [0.173, 0.029, 0.026], + [0.467, 0.053, 0.052], + [1.598, 0.166, 0.162], + [0.54, 0.063, 0.061], + [0.165, 0.037, 0.039], + [1.531, 0.092, 0.09], + [1.494, 0.071, 0.07], + [3.169, 0.224, 0.224], + [0.113, 0.005, 0.005], + [9.03, 0.053, 0.054], + [10.454, 0.066, 0.066], + [20.587, 0.133, 0.134], + [1.514, 0.069, 0.043], + [0.074, 0.013, 0.013], + [0.435, 0.022, 0.017], + [0.101, 0.013, 0.017], + [9.044, 0.049, 0.045], + [7.576, 0.181, 0.187], + [0.048, 0.015, 0.014], + [1.431, 0.057, 0.048], + [4.731, 0.079, 0.077], + [3.501, 0.284, 0.282], + [9.115, 0.19, 0.188], + [9.116, 0.184, 0.182], + [0.136, 0.034, 0.037], + [0.052, 0.014, 0.011], + [0.035, 0.01, 0.01], + [0.044, 0.013, 0.014], + [0.109, 0.034, 0.027], + [0.025, 0.013, 0.008], + [0.022, 0.007, 0.007], + [0.021, 0.009, 0.008] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c6a.xlarge.json b/pivot-parquet-partitioned/results/20261001/c6a.xlarge.json new file mode 100644 index 0000000000..917b469d63 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.006, 0, 0], + [0.054, 0.018, 0.017], + [0.18, 0.076, 0.076], + [0.251, 0.082, 0.082], + [0.817, 0.508, 0.507], + [1.373, 1.035, 1.058], + [0.081, 0.05, 0.05], + [0.057, 0.019, 0.02], + [0.904, 0.774, 0.773], + [1.409, 1.248, 1.251], + [0.343, 0.177, 0.179], + [0.543, 0.25, 0.249], + [1.537, 0.514, 0.606], + [null, null, null], + [0.769, 1.105, 1.008], + [0.569, 0.483, 0.48], + [1.877, 2.177, null], + [1.9, null, null], + [null, null, null], + [0.127, 0.019, 0.019], + [9.019, 7.715, 7.222], + [10.544, 9.102, 8.93], + [20.68, 19.056, 19.082], + [1.614, 0.091, 0.087], + [0.07, 0.016, 0.017], + [0.43, 0.121, 0.121], + [0.087, 0.021, 0.021], + [9.035, 7.558, 7.222], + [7.748, 8.164, 8.207], + [0.16, 0.056, 0.056], + [1.477, 0.741, 0.595], + [4.804, 3.583, 3.811], + [null, null, null], + [null, null, null], + [null, null, null], + [0.574, 0.499, 0.506], + [0.084, 0.029, 0.027], + [0.039, 0.015, 0.014], + [0.069, 0.008, 0.01], + [0.14, 0.049, 0.05], + [0.03, 0.01, 0.011], + [0.023, 0.01, 0.01], + [0.025, 0.014, 0.013] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c7a.metal-48xl.json b/pivot-parquet-partitioned/results/20261001/c7a.metal-48xl.json new file mode 100644 index 0000000000..dfa3ef1faa --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.008, 0.001, 0.001], + [0.014, 0.006, 0.005], + [0.052, 0.006, 0.004], + [0.239, 0.006, 0.005], + [0.266, 0.022, 0.02], + [0.651, 0.036, 0.028], + [0.014, 0.004, 0.004], + [0.021, 0.007, 0.007], + [1.235, 0.036, 0.03], + [0.521, 0.051, 0.049], + [0.208, 0.018, 0.014], + [0.284, 0.023, 0.015], + [0.552, 0.037, 0.024], + [2.212, 0.057, 0.052], + [1.257, 0.037, 0.029], + [0.246, 0.032, 0.024], + [1.511, 0.044, 0.043], + [1.506, 0.039, 0.039], + [3.127, 0.094, 0.098], + [0.181, 0.005, 0.004], + [9.058, 0.021, 0.021], + [10.493, 0.03, 0.029], + [20.631, 0.051, 0.047], + [1.58, 0.032, 0.034], + [0.108, 0.012, 0.009], + [0.475, 0.019, 0.015], + [0.117, 0.009, 0.013], + [9.065, 0.025, 0.025], + [7.565, 0.098, 0.1], + [0.043, 0.016, 0.011], + [1.429, 0.038, 0.026], + [4.733, 0.036, 0.04], + [3.371, 0.117, 0.116], + [9.068, 0.063, 0.062], + [9.071, 0.067, 0.066], + [0.129, 0.029, 0.021], + [0.052, 0.012, 0.009], + [0.026, 0.011, 0.007], + [0.042, 0.009, 0.008], + [0.092, 0.022, 0.02], + [0.034, 0.008, 0.006], + [0.021, 0.007, 0.006], + [0.021, 0.007, 0.007] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c8g.4xlarge.json b/pivot-parquet-partitioned/results/20261001/c8g.4xlarge.json new file mode 100644 index 0000000000..6e5cd43eb0 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.004, 0, 0], + [0.016, 0.004, 0.005], + [0.043, 0.012, 0.011], + [0.211, 0.011, 0.011], + [0.796, 0.081, 0.085], + [1.066, 0.102, 0.102], + [0.016, 0.004, 0.004], + [0.017, 0.005, 0.005], + [0.511, 0.113, 0.11], + [1.143, 0.168, 0.168], + [0.645, 0.031, 0.03], + [0.865, 0.04, 0.039], + [1.209, 0.058, 0.059], + [2.297, 0.149, 0.147], + [1.047, 0.067, 0.066], + [0.755, 0.069, 0.068], + [2.185, 0.135, 0.134], + [2.115, 0.131, 0.131], + [3.893, 0.347, 0.348], + [0.138, 0.004, 0.004], + [10.035, 0.084, 0.084], + [11.319, 0.09, 0.09], + [21.618, 0.203, 0.244], + [2.049, 0.024, 0.025], + [0.096, 0.005, 0.005], + [1.246, 0.021, 0.021], + [0.182, 0.006, 0.006], + [9.995, 0.099, 0.099], + [8.3, 0.574, 0.576], + [0.042, 0.009, 0.009], + [1.505, 0.072, 0.071], + [5.529, 0.09, 0.089], + [4.102, 0.311, 0.312], + [9.437, 0.274, 0.273], + [9.611, 0.276, 0.276], + [0.102, 0.068, 0.069], + [0.037, 0.006, 0.006], + [0.018, 0.004, 0.004], + [0.032, 0.003, 0.003], + [0.069, 0.013, 0.013], + [0.015, 0.003, 0.003], + [0.015, 0.003, 0.003], + [0.012, 0.003, 0.003] +] + } + diff --git a/pivot-parquet-partitioned/results/20261001/c8g.metal-48xl.json b/pivot-parquet-partitioned/results/20261001/c8g.metal-48xl.json new file mode 100644 index 0000000000..2f9b4d0b29 --- /dev/null +++ b/pivot-parquet-partitioned/results/20261001/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "date": "2026-10-01", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.005, 0.001, 0.001], + [0.01, 0.005, 0.004], + [0.047, 0.004, 0.007], + [0.141, 0.004, 0.006], + [0.603, 0.023, 0.022], + [0.915, 0.022, 0.022], + [0.011, 0.004, 0.003], + [0.012, 0.006, 0.006], + [0.303, 0.028, 0.024], + [0.998, 0.039, 0.041], + [0.54, 0.012, 0.012], + [0.605, 0.013, 0.014], + [1.028, 0.024, 0.022], + [2.014, 0.061, 0.059], + [0.934, 0.027, 0.026], + [0.101, 0.016, 0.017], + [1.488, 0.031, 0.029], + [1.476, 0.027, 0.026], + [3.092, 0.081, 0.081], + [0.096, 0.006, 0.004], + [9.021, 0.02, 0.02], + [10.441, 0.023, 0.023], + [20.562, 0.053, 0.045], + [1.508, 0.025, 0.026], + [0.057, 0.007, 0.007], + [0.43, 0.012, 0.01], + [0.083, 0.007, 0.006], + [9.034, 0.02, 0.022], + [7.555, 0.097, 0.092], + [0.034, 0.009, 0.009], + [1.416, 0.025, 0.022], + [4.711, 0.039, 0.031], + [3.35, 0.1, 0.106], + [9.063, 0.072, 0.072], + [9.066, 0.071, 0.075], + [0.066, 0.016, 0.015], + [0.03, 0.007, 0.006], + [0.024, 0.005, 0.005], + [0.027, 0.006, 0.007], + [0.053, 0.016, 0.019], + [0.017, 0.008, 0.007], + [0.015, 0.005, 0.005], + [0.014, 0.005, 0.006] +] + } + diff --git a/pivot-parquet-partitioned/start b/pivot-parquet-partitioned/start new file mode 100755 index 0000000000..f3e03311cd --- /dev/null +++ b/pivot-parquet-partitioned/start @@ -0,0 +1,25 @@ +#!/bin/bash +set -eu +: "${PIVOT_HOME:=$HOME/.pivot}" +: "${PIVOT_PORT:=5432}" + +BIN="$PIVOT_HOME/cli/latest/pivot" + +if psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -c 'SELECT 1' >/dev/null 2>&1; then + exit 0 +fi + +mkdir -p catalog + +cat > config.yaml < server.log 2>&1 & +echo $! > server.pid diff --git a/pivot-parquet-partitioned/stop b/pivot-parquet-partitioned/stop new file mode 100755 index 0000000000..bab361db6c --- /dev/null +++ b/pivot-parquet-partitioned/stop @@ -0,0 +1,19 @@ +#!/bin/bash +set -eu + +[ -f server.pid ] || exit 0 +pid=$(cat server.pid) + +kill "$pid" 2>/dev/null || true + +# The server can hold most of the machine's memory, which only comes back once +# the process is gone; a successor started before that risks an OOM kill. +for _ in $(seq 1 300); do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f server.pid + exit 0 + fi + sleep 0.2 +done +echo "stop: pivot (pid $pid) is still running after 60s" >&2 +exit 1 diff --git a/pivot-parquet-partitioned/template.json b/pivot-parquet-partitioned/template.json new file mode 100644 index 0000000000..5a785271c4 --- /dev/null +++ b/pivot-parquet-partitioned/template.json @@ -0,0 +1,7 @@ +{ + "system": "Pivotlake (Parquet, partitioned)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"] +} diff --git a/pivot-parquet/benchmark.sh b/pivot-parquet/benchmark.sh new file mode 100755 index 0000000000..93e24f228d --- /dev/null +++ b/pivot-parquet/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" +exec ../lib/benchmark-common.sh diff --git a/pivot-parquet/check b/pivot-parquet/check new file mode 100755 index 0000000000..617138474f --- /dev/null +++ b/pivot-parquet/check @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +: "${PIVOT_PORT:=5432}" + +psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -c 'SELECT 1' >/dev/null diff --git a/pivot-parquet/create.sql b/pivot-parquet/create.sql new file mode 100644 index 0000000000..e492f36d7e --- /dev/null +++ b/pivot-parquet/create.sql @@ -0,0 +1,117 @@ +-- Pivot-flavoured ClickBench schema. Maps the original ClickBench schema onto +-- the SQL types pivot's planner understands (BIGINT, INTEGER, SMALLINT, +-- VARCHAR). EventTime/EventDate are stored in the source parquet as packed +-- epoch seconds / days, so they're declared as their integer storage type here; +-- the queries read them as real dates on demand via +-- `make_timestamp(EventTime * 1000000)` (the function counts microseconds, as it +-- does in DuckDB, while the column counts seconds) and `make_date(EventDate)`. CHAR(...) columns are widened to VARCHAR. +-- +-- `load` substitutes `{source}` with the directory holding hits.parquet. +DROP TABLE IF EXISTS hits; +CREATE TABLE hits ( + WatchID BIGINT, + JavaEnable SMALLINT, + Title VARCHAR, + GoodEvent SMALLINT, + EventTime BIGINT, + EventDate USMALLINT, + CounterID INTEGER, + ClientIP INTEGER, + RegionID INTEGER, + UserID BIGINT, + CounterClass SMALLINT, + OS SMALLINT, + UserAgent SMALLINT, + URL VARCHAR, + Referer VARCHAR, + IsRefresh SMALLINT, + RefererCategoryID SMALLINT, + RefererRegionID INTEGER, + URLCategoryID SMALLINT, + URLRegionID INTEGER, + ResolutionWidth SMALLINT, + ResolutionHeight SMALLINT, + ResolutionDepth SMALLINT, + FlashMajor SMALLINT, + FlashMinor SMALLINT, + FlashMinor2 VARCHAR, + NetMajor SMALLINT, + NetMinor SMALLINT, + UserAgentMajor SMALLINT, + UserAgentMinor VARCHAR, + CookieEnable SMALLINT, + JavascriptEnable SMALLINT, + IsMobile SMALLINT, + MobilePhone SMALLINT, + MobilePhoneModel VARCHAR, + Params VARCHAR, + IPNetworkID INTEGER, + TraficSourceID SMALLINT, + SearchEngineID SMALLINT, + SearchPhrase VARCHAR, + AdvEngineID SMALLINT, + IsArtifical SMALLINT, + WindowClientWidth SMALLINT, + WindowClientHeight SMALLINT, + ClientTimeZone SMALLINT, + ClientEventTime BIGINT, + SilverlightVersion1 SMALLINT, + SilverlightVersion2 SMALLINT, + SilverlightVersion3 INTEGER, + SilverlightVersion4 SMALLINT, + PageCharset VARCHAR, + CodeVersion INTEGER, + IsLink SMALLINT, + IsDownload SMALLINT, + IsNotBounce SMALLINT, + FUniqID BIGINT, + OriginalURL VARCHAR, + HID INTEGER, + IsOldCounter SMALLINT, + IsEvent SMALLINT, + IsParameter SMALLINT, + DontCountHits SMALLINT, + WithHash SMALLINT, + HitColor VARCHAR, + LocalEventTime BIGINT, + Age SMALLINT, + Sex SMALLINT, + Income SMALLINT, + Interests SMALLINT, + Robotness SMALLINT, + RemoteIP INTEGER, + WindowName INTEGER, + OpenerName INTEGER, + HistoryLength SMALLINT, + BrowserLanguage VARCHAR, + BrowserCountry VARCHAR, + SocialNetwork VARCHAR, + SocialAction VARCHAR, + HTTPError SMALLINT, + SendTiming INTEGER, + DNSTiming INTEGER, + ConnectTiming INTEGER, + ResponseStartTiming INTEGER, + ResponseEndTiming INTEGER, + FetchTiming INTEGER, + SocialSourceNetworkID SMALLINT, + SocialSourcePage VARCHAR, + ParamPrice BIGINT, + ParamOrderID VARCHAR, + ParamCurrency VARCHAR, + ParamCurrencyID SMALLINT, + OpenstatServiceName VARCHAR, + OpenstatCampaignID VARCHAR, + OpenstatAdID VARCHAR, + OpenstatSourceID VARCHAR, + UTMSource VARCHAR, + UTMMedium VARCHAR, + UTMCampaign VARCHAR, + UTMContent VARCHAR, + UTMTerm VARCHAR, + FromTag VARCHAR, + HasGCLID SMALLINT, + RefererHash BIGINT, + URLHash BIGINT, + CLID INTEGER +) WITH (with_pre_existing_parquets = '{source}'); diff --git a/pivot-parquet/data-size b/pivot-parquet/data-size new file mode 100755 index 0000000000..897028e5cd --- /dev/null +++ b/pivot-parquet/data-size @@ -0,0 +1,5 @@ +#!/bin/bash +set -eu + +# No ingestion — the reported size is the parquet the table reads. +stat -c%s data/hits.parquet diff --git a/pivot-parquet/install b/pivot-parquet/install new file mode 100755 index 0000000000..72f68986a0 --- /dev/null +++ b/pivot-parquet/install @@ -0,0 +1,9 @@ +#!/bin/bash +set -eu + +sudo apt-get update -y +sudo apt-get install -y curl postgresql-client + +# The official installer: it puts the `pivot` binary under $HOME/.pivot (or +# $PIVOT_HOME) and needs no root. ./start runs it from there. +curl https://pivotlake.io | sh diff --git a/pivot-parquet/load b/pivot-parquet/load new file mode 100755 index 0000000000..ce4ae3d5fb --- /dev/null +++ b/pivot-parquet/load @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu +: "${PIVOT_PORT:=5432}" + +# No ingestion: the table adopts the parquet in place, so the file the driver +# downloaded has to stay around for the whole run. +mkdir -p data +mv hits.parquet data/ + +sed "s|{source}|$PWD/data|g" create.sql | + psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -v ON_ERROR_STOP=1 -f - + +sync diff --git a/pivot-parquet/make-json.sh b/pivot-parquet/make-json.sh new file mode 100755 index 0000000000..8c27e1d3e5 --- /dev/null +++ b/pivot-parquet/make-json.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Turns a finished run into results//.json. +# +# ./benchmark.sh +# ./make-json.sh c6a.4xlarge +# +# The per-try timings come from result.csv, which the driver writes as +# ,,; the metadata comes from template.json. The table +# adopts the downloaded parquet in place, so the load is a catalog write and +# the data size is the file's own. +set -eu + +machine=${1:?usage: $0 } +date=$(date -u +%Y-%m-%d) +output="results/${date//-/}/$machine.json" + +mkdir -p "$(dirname "$output")" +jq -n \ + --slurpfile template template.json \ + --arg date "$date" \ + --arg machine "$machine" \ + --rawfile csv result.csv \ + '$template[0] + { + date: $date, + machine: $machine, + cluster_size: 1, + load_time: 0, + data_size: 14779976446, + result: ($csv | split("\n") | map(select(length > 0) | split(",")) + | group_by(.[0] | tonumber) | sort_by(.[0][0] | tonumber) + | map(map(.[2] | if . == "null" then null else tonumber end))) + }' > "$output" +echo "$output" diff --git a/pivot-parquet/queries.sql b/pivot-parquet/queries.sql new file mode 100644 index 0000000000..dbaebeb5b6 --- /dev/null +++ b/pivot-parquet/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0 +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits +SELECT AVG(UserID) FROM hits +SELECT COUNT(DISTINCT UserID) FROM hits +SELECT COUNT(DISTINCT SearchPhrase) FROM hits +SELECT MIN(EventDate), MAX(EventDate) FROM hits +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10 +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10 +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10 +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10 +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10 +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10 +SELECT UserID, extract(minute FROM make_timestamp(EventTime * 1000000)) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10 +SELECT UserID FROM hits WHERE UserID = 435090932899640449 +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%' +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10 +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10 +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25 +SELECT REGEXP_JIT_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25 +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10 +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10 +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10 +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10 +SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10 +SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000 +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000 +SELECT URLHash, make_date(EventDate) AS EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, make_date(EventDate) ORDER BY PageViews DESC LIMIT 10 OFFSET 100 +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-01' AND make_date(EventDate) <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000 +SELECT DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND make_date(EventDate) >= '2013-07-14' AND make_date(EventDate) <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) ORDER BY DATE_TRUNC('minute', make_timestamp(EventTime * 1000000)) LIMIT 10 OFFSET 1000 diff --git a/pivot-parquet/query b/pivot-parquet/query new file mode 100755 index 0000000000..01f4f6279c --- /dev/null +++ b/pivot-parquet/query @@ -0,0 +1,27 @@ +#!/bin/bash +# Reads a SQL query from stdin and runs it via psql against Pivot. +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line (parsed from +# psql's `\timing` "Time: ms" output). +# Exit non-zero on error. +set -e +: "${PIVOT_PORT:=5432}" + +query=$(cat) + +raw=$(psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -v ON_ERROR_STOP=1 -t -c '\timing' -c "$query" 2>&1) && exit_code=0 || exit_code=$? + +if [ "$exit_code" -ne 0 ] || printf '%s\n' "$raw" | grep -qE '^(ERROR|FATAL|PANIC):|psql: error'; then + printf '%s\n' "$raw" >&2 + exit 1 +fi + +printf '%s\n' "$raw" | grep -v '^Time:' + +ms=$(printf '%s\n' "$raw" | grep -oP 'Time:\s*\K[0-9.]+' | tail -n1) +if [ -z "$ms" ]; then + echo "no Time: in psql output" >&2 + exit 1 +fi + +awk -v m="$ms" 'BEGIN { printf "%.3f\n", m / 1000 }' >&2 diff --git a/pivot-parquet/results/20261001/c6a.2xlarge.json b/pivot-parquet/results/20261001/c6a.2xlarge.json new file mode 100644 index 0000000000..36279ad425 --- /dev/null +++ b/pivot-parquet/results/20261001/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.003, 0, 0], + [0.024, 0.011, 0.01], + [0.081, 0.039, 0.04], + [0.157, 0.043, 0.042], + [0.35, 0.306, 0.306], + [0.59, 0.464, 0.466], + [0.033, 0.026, 0.026], + [0.031, 0.013, 0.013], + [0.511, 0.448, 0.453], + [0.813, 0.696, 0.696], + [0.171, 0.098, 0.097], + [0.214, 0.132, 0.131], + [0.438, 0.184, 0.185], + [1.788, 0.526, 0.532], + [0.511, 0.217, 0.217], + [0.302, 0.256, 0.259], + [1.48, 0.677, 0.578], + [1.478, 0.557, 0.563], + [3.569, 1.499, 1.491], + [0.143, 0.013, 0.012], + [9.125, 0.475, 0.54], + [10.579, 0.611, 0.597], + [20.875, 12.68, 10.305], + [2.014, 0.071, 0.071], + [0.05, 0.015, 0.014], + [0.434, 0.066, 0.065], + [0.057, 0.017, 0.017], + [9.137, 0.572, 0.684], + [7.688, 2.131, 2.153], + [0.074, 0.03, 0.029], + [1.414, 0.264, 0.255], + [4.725, 0.313, 0.314], + [3.804, 1.055, 1.049], + [9.149, 8.351, 7.489], + [9.147, 8.29, 7.348], + [0.306, 0.277, 0.272], + [0.05, 0.019, 0.02], + [0.027, 0.013, 0.012], + [0.038, 0.009, 0.008], + [0.094, 0.04, 0.042], + [0.018, 0.009, 0.009], + [0.019, 0.009, 0.01], + [0.017, 0.012, 0.01] +] + } + diff --git a/pivot-parquet/results/20261001/c6a.4xlarge.json b/pivot-parquet/results/20261001/c6a.4xlarge.json new file mode 100644 index 0000000000..2633bea50d --- /dev/null +++ b/pivot-parquet/results/20261001/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.002, 0, 0], + [0.016, 0.009, 0.009], + [0.044, 0.022, 0.022], + [0.287, 0.026, 0.025], + [0.235, 0.173, 0.172], + [0.574, 0.266, 0.265], + [0.02, 0.015, 0.015], + [0.018, 0.011, 0.01], + [0.435, 0.277, 0.273], + [0.714, 0.45, 0.45], + [0.24, 0.063, 0.064], + [0.24, 0.078, 0.078], + [0.44, 0.165, 0.163], + [1.694, 0.407, 0.409], + [0.51, 0.194, 0.193], + [0.21, 0.15, 0.15], + [1.477, 0.368, 0.363], + [1.472, 0.345, 0.346], + [3.474, 1.066, 1.061], + [0.161, 0.012, 0.01], + [9.141, 0.279, 0.274], + [10.562, 0.332, 0.329], + [20.98, 0.772, 0.91], + [2.013, 0.075, 0.075], + [0.147, 0.015, 0.015], + [0.436, 0.039, 0.039], + [0.113, 0.016, 0.016], + [9.132, 0.256, 0.256], + [7.668, 1.081, 1.081], + [0.066, 0.018, 0.018], + [1.413, 0.176, 0.176], + [4.729, 0.276, 0.277], + [4.143, 1.274, 1.272], + [9.133, 0.851, 0.85], + [9.14, 0.855, 0.849], + [0.184, 0.159, 0.158], + [0.041, 0.02, 0.02], + [0.024, 0.011, 0.013], + [0.062, 0.013, 0.011], + [0.11, 0.042, 0.041], + [0.022, 0.01, 0.01], + [0.017, 0.006, 0.009], + [0.014, 0.011, 0.013] +] + } + diff --git a/pivot-parquet/results/20261001/c6a.metal.json b/pivot-parquet/results/20261001/c6a.metal.json new file mode 100644 index 0000000000..1460ab7d12 --- /dev/null +++ b/pivot-parquet/results/20261001/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.004, 0.002, 0.001], + [0.011, 0.005, 0.008], + [0.034, 0.007, 0.006], + [0.141, 0.01, 0.009], + [0.218, 0.044, 0.042], + [0.465, 0.05, 0.055], + [0.008, 0.005, 0.007], + [0.011, 0.006, 0.009], + [0.296, 0.057, 0.062], + [0.51, 0.099, 0.096], + [0.175, 0.02, 0.021], + [0.233, 0.024, 0.028], + [0.467, 0.05, 0.052], + [1.6, 0.165, 0.168], + [0.543, 0.061, 0.056], + [0.175, 0.037, 0.036], + [1.526, 0.092, 0.086], + [1.491, 0.073, 0.07], + [3.135, 0.224, 0.222], + [0.116, 0.005, 0.004], + [9.132, 0.057, 0.051], + [10.561, 0.066, 0.065], + [20.877, 0.133, 0.132], + [2.058, 0.113, 0.07], + [0.113, 0.016, 0.019], + [0.435, 0.025, 0.02], + [0.107, 0.012, 0.018], + [9.14, 0.045, 0.045], + [7.659, 0.196, 0.194], + [0.039, 0.016, 0.018], + [1.437, 0.061, 0.051], + [4.674, 0.078, 0.076], + [3.417, 0.274, 0.271], + [9.224, 0.196, 0.195], + [9.226, 0.196, 0.192], + [0.183, 0.04, 0.036], + [0.039, 0.015, 0.013], + [0.02, 0.01, 0.01], + [0.04, 0.01, 0.018], + [0.097, 0.029, 0.033], + [0.02, 0.01, 0.008], + [0.017, 0.01, 0.012], + [0.016, 0.01, 0.008] +] + } + diff --git a/pivot-parquet/results/20261001/c6a.xlarge.json b/pivot-parquet/results/20261001/c6a.xlarge.json new file mode 100644 index 0000000000..df9c447b54 --- /dev/null +++ b/pivot-parquet/results/20261001/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.002, 0, 0], + [0.049, 0.017, 0.017], + [0.168, 0.076, 0.076], + [0.274, 0.081, 0.081], + [0.584, 0.51, 0.508], + [1.14, 1.048, 1.042], + [0.078, 0.05, 0.05], + [0.051, 0.019, 0.019], + [0.882, 0.772, 0.767], + [1.404, 1.23, 1.233], + [0.366, 0.176, 0.176], + [0.396, 0.243, 0.245], + [0.581, 0.683, 0.736], + [null, null, null], + [0.655, 0.74, 0.786], + [0.545, 0.47, 0.471], + [1.531, 1.748, 1.821], + [null, 1.505, 1.757], + [null, null, null], + [0.21, 0.021, 0.021], + [9.127, 7.896, 7.418], + [10.634, 9.163, 9.176], + [20.931, 19.529, 19.468], + [2.015, 0.186, 0.104], + [0.069, 0.018, 0.018], + [0.433, 0.122, 0.122], + [0.077, 0.023, 0.023], + [9.143, 7.668, 7.439], + [7.781, 8.222, 8.287], + [0.145, 0.056, 0.056], + [1.485, 0.726, 0.806], + [4.74, 3.578, 3.694], + [null, null, null], + [null, null, null], + [null, null, null], + [0.548, 0.484, 0.499], + [0.072, 0.028, 0.028], + [0.032, 0.017, 0.016], + [0.049, 0.01, 0.01], + [0.148, 0.046, 0.048], + [0.022, 0.01, 0.01], + [0.022, 0.011, 0.011], + [0.022, 0.015, 0.015] +] + } + diff --git a/pivot-parquet/results/20261001/c7a.metal-48xl.json b/pivot-parquet/results/20261001/c7a.metal-48xl.json new file mode 100644 index 0000000000..62e22864c2 --- /dev/null +++ b/pivot-parquet/results/20261001/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.003, 0.001, 0.001], + [0.009, 0.005, 0.004], + [0.051, 0.005, 0.005], + [0.177, 0.005, 0.005], + [0.237, 0.025, 0.019], + [0.491, 0.036, 0.026], + [0.008, 0.004, 0.004], + [0.01, 0.006, 0.009], + [0.292, 0.044, 0.031], + [0.485, 0.055, 0.047], + [0.16, 0.019, 0.016], + [0.181, 0.014, 0.017], + [0.461, 0.03, 0.022], + [1.523, 0.051, 0.054], + [0.539, 0.036, 0.024], + [0.168, 0.033, 0.02], + [1.501, 0.042, 0.044], + [1.487, 0.038, 0.038], + [3.069, 0.097, 0.091], + [0.132, 0.004, 0.004], + [9.127, 0.022, 0.025], + [10.559, 0.03, 0.031], + [20.869, 0.058, 0.054], + [2.014, 0.044, 0.028], + [0.123, 0.008, 0.012], + [0.436, 0.012, 0.016], + [0.085, 0.012, 0.008], + [9.14, 0.029, 0.026], + [7.629, 0.1, 0.1], + [0.041, 0.016, 0.014], + [1.435, 0.038, 0.031], + [4.657, 0.043, 0.036], + [3.301, 0.122, 0.118], + [9.171, 0.069, 0.068], + [9.174, 0.065, 0.07], + [0.141, 0.03, 0.029], + [0.063, 0.01, 0.009], + [0.029, 0.009, 0.008], + [0.037, 0.009, 0.011], + [0.076, 0.023, 0.02], + [0.02, 0.007, 0.008], + [0.018, 0.006, 0.006], + [0.015, 0.007, 0.007] +] + } + diff --git a/pivot-parquet/results/20261001/c8g.4xlarge.json b/pivot-parquet/results/20261001/c8g.4xlarge.json new file mode 100644 index 0000000000..68d67dac02 --- /dev/null +++ b/pivot-parquet/results/20261001/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.002, 0, 0], + [0.014, 0.004, 0.005], + [0.034, 0.011, 0.011], + [0.234, 0.011, 0.011], + [0.283, 0.082, 0.084], + [0.453, 0.103, 0.103], + [0.011, 0.004, 0.004], + [0.013, 0.005, 0.005], + [0.304, 0.113, 0.11], + [0.502, 0.169, 0.172], + [0.177, 0.032, 0.03], + [0.15, 0.041, 0.039], + [0.436, 0.058, 0.059], + [1.56, 0.157, 0.151], + [0.505, 0.07, 0.068], + [0.175, 0.068, 0.068], + [1.467, 0.135, 0.135], + [1.468, 0.131, 0.131], + [3.158, 0.351, 0.351], + [0.112, 0.005, 0.005], + [9.125, 0.087, 0.086], + [10.554, 0.093, 0.093], + [20.866, 0.217, 0.298], + [2.009, 0.026, 0.026], + [0.062, 0.005, 0.005], + [0.431, 0.022, 0.021], + [0.062, 0.005, 0.005], + [9.153, 0.103, 0.103], + [7.63, 0.577, 0.578], + [0.035, 0.01, 0.01], + [1.41, 0.074, 0.073], + [4.656, 0.093, 0.089], + [3.407, 0.314, 0.319], + [9.13, 0.282, 0.282], + [9.132, 0.285, 0.283], + [0.161, 0.069, 0.07], + [0.039, 0.006, 0.006], + [0.015, 0.004, 0.004], + [0.046, 0.003, 0.003], + [0.07, 0.012, 0.013], + [0.014, 0.003, 0.003], + [0.012, 0.003, 0.003], + [0.01, 0.003, 0.004] +] + } + diff --git a/pivot-parquet/results/20261001/c8g.metal-48xl.json b/pivot-parquet/results/20261001/c8g.metal-48xl.json new file mode 100644 index 0000000000..fb74586a79 --- /dev/null +++ b/pivot-parquet/results/20261001/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Pivotlake (Parquet)", + "date": "2026-10-01", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"], + "load_time": 0, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.003, 0.001, 0.001], + [0.007, 0.004, 0.004], + [0.037, 0.004, 0.004], + [0.366, 0.005, 0.004], + [0.202, 0.026, 0.021], + [0.444, 0.023, 0.025], + [0.007, 0.003, 0.007], + [0.008, 0.009, 0.008], + [0.278, 0.024, 0.024], + [0.478, 0.043, 0.042], + [0.147, 0.011, 0.011], + [0.164, 0.016, 0.014], + [0.45, 0.021, 0.023], + [1.516, 0.06, 0.059], + [0.523, 0.029, 0.025], + [0.128, 0.015, 0.018], + [1.49, 0.035, 0.034], + [1.478, 0.026, 0.027], + [3.053, 0.081, 0.079], + [0.215, 0.003, 0.007], + [9.127, 0.02, 0.023], + [10.561, 0.025, 0.029], + [20.867, 0.046, 0.048], + [2.013, 0.027, 0.024], + [0.157, 0.008, 0.009], + [1.387, 0.011, 0.011], + [0.069, 0.007, 0.007], + [9.135, 0.023, 0.023], + [7.623, 0.097, 0.094], + [0.035, 0.013, 0.009], + [1.423, 0.027, 0.024], + [4.647, 0.039, 0.035], + [3.276, 0.096, 0.099], + [9.169, 0.076, 0.074], + [9.169, 0.073, 0.072], + [0.109, 0.018, 0.019], + [0.029, 0.01, 0.007], + [0.019, 0.005, 0.006], + [0.058, 0.005, 0.005], + [0.08, 0.013, 0.018], + [0.015, 0.005, 0.005], + [0.013, 0.007, 0.005], + [0.012, 0.006, 0.01] +] + } + diff --git a/pivot-parquet/start b/pivot-parquet/start new file mode 100755 index 0000000000..f3e03311cd --- /dev/null +++ b/pivot-parquet/start @@ -0,0 +1,25 @@ +#!/bin/bash +set -eu +: "${PIVOT_HOME:=$HOME/.pivot}" +: "${PIVOT_PORT:=5432}" + +BIN="$PIVOT_HOME/cli/latest/pivot" + +if psql -h 127.0.0.1 -p "$PIVOT_PORT" -U pivot -d postgres -c 'SELECT 1' >/dev/null 2>&1; then + exit 0 +fi + +mkdir -p catalog + +cat > config.yaml < server.log 2>&1 & +echo $! > server.pid diff --git a/pivot-parquet/stop b/pivot-parquet/stop new file mode 100755 index 0000000000..bab361db6c --- /dev/null +++ b/pivot-parquet/stop @@ -0,0 +1,19 @@ +#!/bin/bash +set -eu + +[ -f server.pid ] || exit 0 +pid=$(cat server.pid) + +kill "$pid" 2>/dev/null || true + +# The server can hold most of the machine's memory, which only comes back once +# the process is gone; a successor started before that risks an OOM kill. +for _ in $(seq 1 300); do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f server.pid + exit 0 + fi + sleep 0.2 +done +echo "stop: pivot (pid $pid) is still running after 60s" >&2 +exit 1 diff --git a/pivot-parquet/template.json b/pivot-parquet/template.json new file mode 100644 index 0000000000..719cd9ec2e --- /dev/null +++ b/pivot-parquet/template.json @@ -0,0 +1,7 @@ +{ + "system": "Pivotlake (Parquet)", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","PostgreSQL compatible","stateless"] +}