From 815f253ba2e0e06fbd0c78603c2349795fc3d5e6 Mon Sep 17 00:00:00 2001 From: Satya Patel Date: Thu, 4 Jun 2026 17:12:25 -0700 Subject: [PATCH 1/5] feat(node-db-sqlite-persistence): prune applied_tx by default createNodeSQLitePersistence left appliedTxPruneMaxRows/appliedTxPruneMaxAgeSeconds unset, so the applied_tx log grew without bound for every consumer. Default them to 1,000 rows and a 24h age backstop (both overridable; pass 0 to disable), exported as DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS / DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS. --- .changeset/node-default-applied-tx-pruning.md | 5 + packages/node-db-sqlite-persistence/README.md | 24 ++++ .../node-db-sqlite-persistence/src/index.ts | 6 +- .../src/node-persistence.ts | 21 +++- .../tests/node-persistence.test.ts | 115 ++++++++++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 .changeset/node-default-applied-tx-pruning.md diff --git a/.changeset/node-default-applied-tx-pruning.md b/.changeset/node-default-applied-tx-pruning.md new file mode 100644 index 000000000..c313ab306 --- /dev/null +++ b/.changeset/node-default-applied-tx-pruning.md @@ -0,0 +1,5 @@ +--- +'@tanstack/node-db-sqlite-persistence': minor +--- + +`createNodeSQLitePersistence` now prunes the `applied_tx` log by default so the SQLite file no longer grows without bound. When prune options are omitted, the node driver applies `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`. diff --git a/packages/node-db-sqlite-persistence/README.md b/packages/node-db-sqlite-persistence/README.md index fc2fdb2f6..25374f391 100644 --- a/packages/node-db-sqlite-persistence/README.md +++ b/packages/node-db-sqlite-persistence/README.md @@ -47,3 +47,27 @@ export const todosCollection = createCollection( mode-specific behavior (`sync-present` vs `sync-absent`) automatically. - `schemaVersion` is specified per collection via `persistedCollectionOptions`. - Call `database.close()` when your app shuts down. + +## Applied transaction pruning + +The `applied_tx` log is a replayable cache, so it is pruned by default to keep +the SQLite file from growing without bound. When you don't pass prune options, +the node driver applies: + +- `appliedTxPruneMaxRows: 1_000` (per-collection row cap) +- `appliedTxPruneMaxAgeSeconds: 86_400` (24h age backstop) + +Pruning runs inside each write transaction, so every collection self-trims on +its next sync. Override either value to tune retention, or pass `0` to disable +that limit: + +```ts +const persistence = createNodeSQLitePersistence({ + database, + appliedTxPruneMaxRows: 5_000, // higher row cap + appliedTxPruneMaxAgeSeconds: 0, // disable the age backstop +}) +``` + +The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and +`DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`. diff --git a/packages/node-db-sqlite-persistence/src/index.ts b/packages/node-db-sqlite-persistence/src/index.ts index 52c936c1d..0a3a703b3 100644 --- a/packages/node-db-sqlite-persistence/src/index.ts +++ b/packages/node-db-sqlite-persistence/src/index.ts @@ -1,4 +1,8 @@ -export { createNodeSQLitePersistence } from './node-persistence' +export { + createNodeSQLitePersistence, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, +} from './node-persistence' export type { BetterSqlite3Database, NodeSQLitePersistenceOptions, diff --git a/packages/node-db-sqlite-persistence/src/node-persistence.ts b/packages/node-db-sqlite-persistence/src/node-persistence.ts index 2dbd718c3..9379ada65 100644 --- a/packages/node-db-sqlite-persistence/src/node-persistence.ts +++ b/packages/node-db-sqlite-persistence/src/node-persistence.ts @@ -35,6 +35,20 @@ type NodeSQLitePersistenceBaseOptions = Omit< export type NodeSQLitePersistenceOptions = NodeSQLitePersistenceBaseOptions +/** + * Default cap on retained `applied_tx` rows per collection. The log is a + * replayable cache, so a bounded row count keeps the SQLite file from growing + * without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap. + */ +export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = 1_000 + +/** + * Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows + * older than this are pruned on the next write. Pass + * `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop. + */ +export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60 + function normalizeSchemaMismatchPolicy( policy: NodeSQLiteSchemaMismatchPolicy, ): NodeSQLiteCoreSchemaMismatchPolicy { @@ -81,8 +95,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts index 12fc53f93..fa8bb5853 100644 --- a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts +++ b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts @@ -127,6 +127,121 @@ describe(`node persistence helpers`, () => { } }) + it(`prunes applied_tx rows past the default age backstop`, async () => { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-default-prune-`)) + const dbPath = join(tempDirectory, `state.sqlite`) + const collectionId = `default-prune` + const database = new BetterSqlite3(dbPath) + + try { + const persistence = createNodeSQLitePersistence({ database }) + + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `1`, + value: { id: `1`, title: `old`, score: 1 }, + }, + ], + }) + + // Backdate the first row well beyond the 24h default age backstop. + database + .prepare( + `UPDATE applied_tx SET applied_at = 0 WHERE collection_id = ? AND seq = 1`, + ) + .run(collectionId) + + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-2`, + term: 1, + seq: 2, + rowVersion: 2, + mutations: [ + { + type: `insert`, + key: `2`, + value: { id: `2`, title: `new`, score: 2 }, + }, + ], + }) + + const appliedRows = database + .prepare( + `SELECT seq FROM applied_tx WHERE collection_id = ? ORDER BY seq ASC`, + ) + .all(collectionId) as Array<{ seq: number }> + expect(appliedRows.map((row) => row.seq)).toEqual([2]) + } finally { + database.close() + rmSync(tempDirectory, { recursive: true, force: true }) + } + }) + + it(`leaves applied_tx rows untouched when pruning is disabled`, async () => { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-no-prune-`)) + const dbPath = join(tempDirectory, `state.sqlite`) + const collectionId = `no-prune` + const database = new BetterSqlite3(dbPath) + + try { + const persistence = createNodeSQLitePersistence({ + database, + appliedTxPruneMaxRows: 0, + appliedTxPruneMaxAgeSeconds: 0, + }) + + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `1`, + value: { id: `1`, title: `old`, score: 1 }, + }, + ], + }) + + database + .prepare( + `UPDATE applied_tx SET applied_at = 0 WHERE collection_id = ? AND seq = 1`, + ) + .run(collectionId) + + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-2`, + term: 1, + seq: 2, + rowVersion: 2, + mutations: [ + { + type: `insert`, + key: `2`, + value: { id: `2`, title: `new`, score: 2 }, + }, + ], + }) + + const appliedRows = database + .prepare( + `SELECT seq FROM applied_tx WHERE collection_id = ? ORDER BY seq ASC`, + ) + .all(collectionId) as Array<{ seq: number }> + expect(appliedRows.map((row) => row.seq)).toEqual([1, 2]) + } finally { + database.close() + rmSync(tempDirectory, { recursive: true, force: true }) + } + }) + it(`infers schema policy from sync mode`, async () => { const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-schema-infer-`)) const dbPath = join(tempDirectory, `state.sqlite`) From af234434d27e7e5643ba5784500b3f9d68c41f73 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 5 Jun 2026 11:08:18 -0600 Subject: [PATCH 2/5] fix: make sqlite replay pruning recovery-aware --- .../src/sqlite-core-adapter.ts | 26 +++++- packages/node-db-sqlite-persistence/README.md | 9 +- .../tests/node-persistence.test.ts | 86 +++++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 6bbf06c4d..2fe7c642a 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1540,8 +1540,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { const collectionTableSql = quoteIdentifier(tableMapping.tableName) const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) - const [changedRows, deletedRows, latestVersionRows, replayRows] = - await Promise.all([ + const [ + changedRows, + deletedRows, + latestVersionRows, + replayRows, + replayAvailabilityRows, + ] = await Promise.all([ this.driver.query<{ key: string }>( `SELECT key FROM ${collectionTableSql} @@ -1573,9 +1578,26 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ORDER BY term ASC, seq ASC`, [collectionId, fromRowVersion], ), + this.driver.query<{ min_row_version: number | null }>( + `SELECT MIN(row_version) AS min_row_version + FROM applied_tx + WHERE collection_id = ?`, + [collectionId], + ), ]) const latestRowVersion = latestVersionRows[0]?.latest_row_version ?? 0 + const replayFloor = replayAvailabilityRows[0]?.min_row_version + if ( + latestRowVersion > fromRowVersion && + (replayFloor == null || replayFloor > fromRowVersion + 1) + ) { + return { + latestRowVersion, + requiresFullReload: true, + } + } + const changedKeyCount = changedRows.length + deletedRows.length if (changedKeyCount > this.pullSinceReloadThreshold) { diff --git a/packages/node-db-sqlite-persistence/README.md b/packages/node-db-sqlite-persistence/README.md index 25374f391..23fc70836 100644 --- a/packages/node-db-sqlite-persistence/README.md +++ b/packages/node-db-sqlite-persistence/README.md @@ -58,8 +58,9 @@ the node driver applies: - `appliedTxPruneMaxAgeSeconds: 86_400` (24h age backstop) Pruning runs inside each write transaction, so every collection self-trims on -its next sync. Override either value to tune retention, or pass `0` to disable -that limit: +its next sync. If a process asks to recover from a point older than the retained +replay window, recovery falls back to a full reload. Override either value to +tune retention, or pass `0` to disable that limit: ```ts const persistence = createNodeSQLitePersistence({ @@ -69,5 +70,9 @@ const persistence = createNodeSQLitePersistence({ }) ``` +Pruning removes rows from `applied_tx`, but SQLite may not immediately shrink +the database file on disk. Use `VACUUM`, `auto_vacuum`, or your own maintenance +process if you need to reclaim disk space. + The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`. diff --git a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts index fa8bb5853..0ebd8c39e 100644 --- a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts +++ b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts @@ -7,6 +7,7 @@ import { createNodeSQLitePersistence, persistedCollectionOptions } from '../src' import { BetterSqlite3SQLiteDriver } from '../src/node-driver' import { SingleProcessCoordinator } from '../../db-sqlite-persistence-core/src' import { runRuntimePersistenceContractSuite } from '../../db-sqlite-persistence-core/tests/contracts/runtime-persistence-contract' +import type { SQLitePullSinceResult } from '../../db-sqlite-persistence-core/src' import type { RuntimePersistenceContractTodo, RuntimePersistenceDatabaseHarness, @@ -183,6 +184,91 @@ describe(`node persistence helpers`, () => { } }) + it(`forces full reload when pullSince starts before pruned replay rows`, async () => { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-pruned-pull-since-`)) + const dbPath = join(tempDirectory, `state.sqlite`) + const collectionId = `pruned-pull-since` + const database = new BetterSqlite3(dbPath) + + try { + const persistence = createNodeSQLitePersistence({ + database, + appliedTxPruneMaxRows: 2, + appliedTxPruneMaxAgeSeconds: 0, + }) + + for (const seq of [1, 2, 3]) { + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-${seq}`, + term: 1, + seq, + rowVersion: seq, + mutations: [ + { + type: `insert`, + key: String(seq), + value: { id: String(seq), title: `todo-${seq}`, score: seq }, + }, + ], + }) + } + + const adapter = persistence.adapter as typeof persistence.adapter & { + pullSince: ( + collectionId: string, + fromRowVersion: number, + ) => Promise> + } + const result = await adapter.pullSince(collectionId, 0) + + expect(result.requiresFullReload).toBe(true) + } finally { + database.close() + rmSync(tempDirectory, { recursive: true, force: true }) + } + }) + + it(`prunes applied_tx rows past explicit row cap`, async () => { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-row-prune-`)) + const dbPath = join(tempDirectory, `state.sqlite`) + const collectionId = `row-prune` + const database = new BetterSqlite3(dbPath) + + try { + const persistence = createNodeSQLitePersistence({ + database, + appliedTxPruneMaxRows: 2, + appliedTxPruneMaxAgeSeconds: 0, + }) + + for (const seq of [1, 2, 3]) { + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `tx-${seq}`, + term: 1, + seq, + rowVersion: seq, + mutations: [ + { + type: `insert`, + key: String(seq), + value: { id: String(seq), title: `todo-${seq}`, score: seq }, + }, + ], + }) + } + + const appliedRows = database + .prepare( + `SELECT seq FROM applied_tx WHERE collection_id = ? ORDER BY seq ASC`, + ) + .all(collectionId) as Array<{ seq: number }> + expect(appliedRows.map((row) => row.seq)).toEqual([2, 3]) + } finally { + database.close() + rmSync(tempDirectory, { recursive: true, force: true }) + } + }) + it(`leaves applied_tx rows untouched when pruning is disabled`, async () => { const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-no-prune-`)) const dbPath = join(tempDirectory, `state.sqlite`) From 768ed1290d7138193fc684749c84b4e67b1dd932 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2026 17:10:12 +0000 Subject: [PATCH 3/5] ci: apply automated fixes --- .../src/sqlite-core-adapter.ts | 52 +++++++++---------- .../tests/node-persistence.test.ts | 4 +- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 2fe7c642a..2414db79c 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1547,44 +1547,44 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { replayRows, replayAvailabilityRows, ] = await Promise.all([ - this.driver.query<{ key: string }>( - `SELECT key + this.driver.query<{ key: string }>( + `SELECT key FROM ${collectionTableSql} WHERE row_version > ?`, - [fromRowVersion], - ), - this.driver.query<{ key: string }>( - `SELECT key + [fromRowVersion], + ), + this.driver.query<{ key: string }>( + `SELECT key FROM ${tombstoneTableSql} WHERE row_version > ?`, - [fromRowVersion], - ), - this.driver.query<{ latest_row_version: number }>( - `SELECT latest_row_version + [fromRowVersion], + ), + this.driver.query<{ latest_row_version: number }>( + `SELECT latest_row_version FROM collection_version WHERE collection_id = ? LIMIT 1`, - [collectionId], - ), - this.driver.query<{ - tx_id: string - row_version: number - replay_json: string | null - replay_requires_full_reload: number - }>( - `SELECT tx_id, row_version, replay_json, replay_requires_full_reload + [collectionId], + ), + this.driver.query<{ + tx_id: string + row_version: number + replay_json: string | null + replay_requires_full_reload: number + }>( + `SELECT tx_id, row_version, replay_json, replay_requires_full_reload FROM applied_tx WHERE collection_id = ? AND row_version > ? ORDER BY term ASC, seq ASC`, - [collectionId, fromRowVersion], - ), - this.driver.query<{ min_row_version: number | null }>( - `SELECT MIN(row_version) AS min_row_version + [collectionId, fromRowVersion], + ), + this.driver.query<{ min_row_version: number | null }>( + `SELECT MIN(row_version) AS min_row_version FROM applied_tx WHERE collection_id = ?`, - [collectionId], - ), - ]) + [collectionId], + ), + ]) const latestRowVersion = latestVersionRows[0]?.latest_row_version ?? 0 const replayFloor = replayAvailabilityRows[0]?.min_row_version diff --git a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts index 0ebd8c39e..15669b7b8 100644 --- a/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts +++ b/packages/node-db-sqlite-persistence/tests/node-persistence.test.ts @@ -185,7 +185,9 @@ describe(`node persistence helpers`, () => { }) it(`forces full reload when pullSince starts before pruned replay rows`, async () => { - const tempDirectory = mkdtempSync(join(tmpdir(), `db-node-pruned-pull-since-`)) + const tempDirectory = mkdtempSync( + join(tmpdir(), `db-node-pruned-pull-since-`), + ) const dbPath = join(tempDirectory, `state.sqlite`) const collectionId = `pruned-pull-since` const database = new BetterSqlite3(dbPath) From 5416538287aac09aec400c5ad296026c3d3ab5b5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 5 Jun 2026 11:32:06 -0600 Subject: [PATCH 4/5] docs: clarify sqlite pruning scope --- .changeset/node-default-applied-tx-pruning.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.changeset/node-default-applied-tx-pruning.md b/.changeset/node-default-applied-tx-pruning.md index c313ab306..84c2634ea 100644 --- a/.changeset/node-default-applied-tx-pruning.md +++ b/.changeset/node-default-applied-tx-pruning.md @@ -1,5 +1,8 @@ --- '@tanstack/node-db-sqlite-persistence': minor +'@tanstack/db-sqlite-persistence-core': patch --- `createNodeSQLitePersistence` now prunes the `applied_tx` log by default so the SQLite file no longer grows without bound. When prune options are omitted, the node driver applies `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`. + +The shared SQLite core adapter now treats `applied_tx` as a bounded replay cache during `pullSince` recovery. If a recovery request starts before the retained replay window, `pullSince` returns `requiresFullReload: true` instead of returning partial deltas. This safety fix applies to every SQLite persistence wrapper that opts into `applied_tx` pruning; this changeset only enables pruning by default for the Node wrapper. From ce97fa4893856c3bc896f58db34cb3b75a94e3ee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 5 Jun 2026 11:39:30 -0600 Subject: [PATCH 5/5] feat: default sqlite applied_tx pruning across wrappers --- .changeset/node-default-applied-tx-pruning.md | 12 +++++++++--- .../src/browser-persistence.ts | 9 +++++++-- .../browser-db-sqlite-persistence/src/index.ts | 6 +++++- .../src/capacitor-persistence.ts | 9 +++++++-- .../capacitor-db-sqlite-persistence/src/index.ts | 6 +++++- .../src/do-persistence.ts | 9 +++++++-- .../src/index.ts | 6 +++++- .../src/sqlite-core-adapter.ts | 15 +++++++++++++++ packages/expo-db-sqlite-persistence/src/expo.ts | 9 +++++++-- packages/expo-db-sqlite-persistence/src/index.ts | 6 +++++- .../src/node-persistence.ts | 14 +++++++++----- .../src/index.ts | 6 +++++- .../src/mobile-persistence.ts | 9 +++++++-- packages/tauri-db-sqlite-persistence/src/index.ts | 6 +++++- .../src/tauri-persistence.ts | 9 +++++++-- 15 files changed, 105 insertions(+), 26 deletions(-) diff --git a/.changeset/node-default-applied-tx-pruning.md b/.changeset/node-default-applied-tx-pruning.md index 84c2634ea..28958ca90 100644 --- a/.changeset/node-default-applied-tx-pruning.md +++ b/.changeset/node-default-applied-tx-pruning.md @@ -1,8 +1,14 @@ --- +'@tanstack/browser-db-sqlite-persistence': minor +'@tanstack/capacitor-db-sqlite-persistence': minor +'@tanstack/cloudflare-durable-objects-db-sqlite-persistence': minor +'@tanstack/db-sqlite-persistence-core': minor +'@tanstack/expo-db-sqlite-persistence': minor '@tanstack/node-db-sqlite-persistence': minor -'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/react-native-db-sqlite-persistence': minor +'@tanstack/tauri-db-sqlite-persistence': minor --- -`createNodeSQLitePersistence` now prunes the `applied_tx` log by default so the SQLite file no longer grows without bound. When prune options are omitted, the node driver applies `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`. +SQLite persistence wrappers now prune the `applied_tx` replay log by default so SQLite files no longer grow without bound. When prune options are omitted, wrappers that construct the shared SQLite core adapter apply `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS` from the shared SQLite core package and re-exported by wrapper packages. -The shared SQLite core adapter now treats `applied_tx` as a bounded replay cache during `pullSince` recovery. If a recovery request starts before the retained replay window, `pullSince` returns `requiresFullReload: true` instead of returning partial deltas. This safety fix applies to every SQLite persistence wrapper that opts into `applied_tx` pruning; this changeset only enables pruning by default for the Node wrapper. +The shared SQLite core adapter now treats `applied_tx` as a bounded replay cache during `pullSince` recovery. If a recovery request starts before the retained replay window, `pullSince` returns `requiresFullReload: true` instead of returning partial deltas. diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index bfa049bf6..b36e56490 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -78,8 +80,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/browser-db-sqlite-persistence/src/index.ts b/packages/browser-db-sqlite-persistence/src/index.ts index 5931b8ade..992e8c092 100644 --- a/packages/browser-db-sqlite-persistence/src/index.ts +++ b/packages/browser-db-sqlite-persistence/src/index.ts @@ -8,7 +8,11 @@ export type { } from './browser-persistence' export type { OpenBrowserWASQLiteOPFSDatabaseOptions } from './opfs-database' export type { BrowserCollectionCoordinatorOptions } from './browser-coordinator' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/capacitor-db-sqlite-persistence/src/capacitor-persistence.ts b/packages/capacitor-db-sqlite-persistence/src/capacitor-persistence.ts index 9eea35b9b..e293ea71b 100644 --- a/packages/capacitor-db-sqlite-persistence/src/capacitor-persistence.ts +++ b/packages/capacitor-db-sqlite-persistence/src/capacitor-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -81,8 +83,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/capacitor-db-sqlite-persistence/src/index.ts b/packages/capacitor-db-sqlite-persistence/src/index.ts index 81f8b6c40..1aad1c786 100644 --- a/packages/capacitor-db-sqlite-persistence/src/index.ts +++ b/packages/capacitor-db-sqlite-persistence/src/index.ts @@ -5,7 +5,11 @@ export type { CapacitorSQLiteSchemaMismatchPolicy, SQLiteDBConnection, } from './capacitor' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/src/do-persistence.ts b/packages/cloudflare-durable-objects-db-sqlite-persistence/src/do-persistence.ts index bf6c7667f..3d5ef2311 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/src/do-persistence.ts +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/src/do-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -79,8 +81,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/src/index.ts b/packages/cloudflare-durable-objects-db-sqlite-persistence/src/index.ts index 5740c803b..714e8912f 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/src/index.ts +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/src/index.ts @@ -4,7 +4,11 @@ export type { CloudflareDOSQLitePersistenceOptions, DurableObjectStorageLike, } from './do-persistence' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 2414db79c..69f29fc60 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -73,6 +73,21 @@ export type SQLitePullSinceResult = const DEFAULT_SCHEMA_VERSION = 1 const DEFAULT_PULL_SINCE_RELOAD_THRESHOLD = 128 + +/** + * Default cap on retained `applied_tx` rows per collection. The log is a + * replayable cache, so a bounded row count keeps SQLite files from growing + * without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap. + */ +export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = 1_000 + +/** + * Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows + * older than this are pruned on the next write. Pass + * `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop. + */ +export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60 + const SQLITE_MAX_IN_BATCH_SIZE = 900 const SAFE_IDENTIFIER_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/ const FORBIDDEN_SQL_FRAGMENT_PATTERN = /(;|--|\/\*)/ diff --git a/packages/expo-db-sqlite-persistence/src/expo.ts b/packages/expo-db-sqlite-persistence/src/expo.ts index 53cf16e0e..bdb490d21 100644 --- a/packages/expo-db-sqlite-persistence/src/expo.ts +++ b/packages/expo-db-sqlite-persistence/src/expo.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -77,8 +79,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/expo-db-sqlite-persistence/src/index.ts b/packages/expo-db-sqlite-persistence/src/index.ts index 6b6968dca..0f23fd540 100644 --- a/packages/expo-db-sqlite-persistence/src/index.ts +++ b/packages/expo-db-sqlite-persistence/src/index.ts @@ -4,7 +4,11 @@ export type { ExpoSQLitePersistenceOptions, ExpoSQLiteSchemaMismatchPolicy, } from './expo' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/node-db-sqlite-persistence/src/node-persistence.ts b/packages/node-db-sqlite-persistence/src/node-persistence.ts index 9379ada65..4635b9c65 100644 --- a/packages/node-db-sqlite-persistence/src/node-persistence.ts +++ b/packages/node-db-sqlite-persistence/src/node-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS as CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS as CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -37,17 +39,19 @@ export type NodeSQLitePersistenceOptions = NodeSQLitePersistenceBaseOptions /** * Default cap on retained `applied_tx` rows per collection. The log is a - * replayable cache, so a bounded row count keeps the SQLite file from growing + * replayable cache, so a bounded row count keeps SQLite files from growing * without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap. */ -export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = 1_000 +export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = + CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS /** * Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows * older than this are pruned on the next write. Pass * `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop. */ -export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60 +export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = + CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS function normalizeSchemaMismatchPolicy( policy: NodeSQLiteSchemaMismatchPolicy, @@ -96,10 +100,10 @@ function resolveAdapterBaseOptions( > { return { appliedTxPruneMaxRows: - options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + options.appliedTxPruneMaxRows ?? CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds ?? - DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/react-native-db-sqlite-persistence/src/index.ts b/packages/react-native-db-sqlite-persistence/src/index.ts index 842298336..8dc1b4aeb 100644 --- a/packages/react-native-db-sqlite-persistence/src/index.ts +++ b/packages/react-native-db-sqlite-persistence/src/index.ts @@ -4,7 +4,11 @@ export type { ReactNativeSQLitePersistenceOptions, ReactNativeSQLiteSchemaMismatchPolicy, } from './react-native' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/react-native-db-sqlite-persistence/src/mobile-persistence.ts b/packages/react-native-db-sqlite-persistence/src/mobile-persistence.ts index 073994f38..7fce7b436 100644 --- a/packages/react-native-db-sqlite-persistence/src/mobile-persistence.ts +++ b/packages/react-native-db-sqlite-persistence/src/mobile-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -79,8 +81,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } } diff --git a/packages/tauri-db-sqlite-persistence/src/index.ts b/packages/tauri-db-sqlite-persistence/src/index.ts index 7a750708d..e9ebfc7f3 100644 --- a/packages/tauri-db-sqlite-persistence/src/index.ts +++ b/packages/tauri-db-sqlite-persistence/src/index.ts @@ -4,7 +4,11 @@ export type { TauriSQLitePersistenceOptions, TauriSQLiteSchemaMismatchPolicy, } from './tauri' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, diff --git a/packages/tauri-db-sqlite-persistence/src/tauri-persistence.ts b/packages/tauri-db-sqlite-persistence/src/tauri-persistence.ts index 956f64dfd..f3dea0d17 100644 --- a/packages/tauri-db-sqlite-persistence/src/tauri-persistence.ts +++ b/packages/tauri-db-sqlite-persistence/src/tauri-persistence.ts @@ -1,4 +1,6 @@ import { + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, + DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, SingleProcessCoordinator, createSQLiteCorePersistenceAdapter, } from '@tanstack/db-sqlite-persistence-core' @@ -91,8 +93,11 @@ function resolveAdapterBaseOptions( `driver` | `schemaVersion` | `schemaMismatchPolicy` > { return { - appliedTxPruneMaxRows: options.appliedTxPruneMaxRows, - appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds, + appliedTxPruneMaxRows: + options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + appliedTxPruneMaxAgeSeconds: + options.appliedTxPruneMaxAgeSeconds ?? + DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, pullSinceReloadThreshold: options.pullSinceReloadThreshold, } }