Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .changeset/node-default-applied-tx-pruning.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
'@tanstack/browser-db-sqlite-persistence': minor
'@tanstack/capacitor-db-sqlite-persistence': minor
'@tanstack/cloudflare-durable-objects-db-sqlite-persistence': minor
'@tanstack/db-sqlite-persistence-core': minor
'@tanstack/expo-db-sqlite-persistence': minor
'@tanstack/node-db-sqlite-persistence': minor
'@tanstack/react-native-db-sqlite-persistence': minor
'@tanstack/tauri-db-sqlite-persistence': minor
---

SQLite persistence wrappers now prune the `applied_tx` replay log by default so SQLite files no longer grow without bound. When prune options are omitted, wrappers that construct the shared SQLite core adapter apply `appliedTxPruneMaxRows: 1_000` and `appliedTxPruneMaxAgeSeconds: 86_400` (24h). Both remain overridable, and passing `0` disables that limit. The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS` from the shared SQLite core package and re-exported by wrapper packages.

The shared SQLite core adapter now treats `applied_tx` as a bounded replay cache during `pullSince` recovery. If a recovery request starts before the retained replay window, `pullSince` returns `requiresFullReload: true` instead of returning partial deltas.
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
SingleProcessCoordinator,
createSQLiteCorePersistenceAdapter,
} from '@tanstack/db-sqlite-persistence-core'
Expand Down Expand Up @@ -78,8 +80,11 @@ function resolveAdapterBaseOptions(
`driver` | `schemaVersion` | `schemaMismatchPolicy`
> {
return {
appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
appliedTxPruneMaxRows:
options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
appliedTxPruneMaxAgeSeconds:
options.appliedTxPruneMaxAgeSeconds ??
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
pullSinceReloadThreshold: options.pullSinceReloadThreshold,
}
}
Expand Down
6 changes: 5 additions & 1 deletion packages/browser-db-sqlite-persistence/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ export type {
} from './browser-persistence'
export type { OpenBrowserWASQLiteOPFSDatabaseOptions } from './opfs-database'
export type { BrowserCollectionCoordinatorOptions } from './browser-coordinator'
export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core'
export {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
persistedCollectionOptions,
} from '@tanstack/db-sqlite-persistence-core'
export type {
PersistedCollectionCoordinator,
PersistedCollectionPersistence,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
SingleProcessCoordinator,
createSQLiteCorePersistenceAdapter,
} from '@tanstack/db-sqlite-persistence-core'
Expand Down Expand Up @@ -81,8 +83,11 @@ function resolveAdapterBaseOptions(
`driver` | `schemaVersion` | `schemaMismatchPolicy`
> {
return {
appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
appliedTxPruneMaxRows:
options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
appliedTxPruneMaxAgeSeconds:
options.appliedTxPruneMaxAgeSeconds ??
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
pullSinceReloadThreshold: options.pullSinceReloadThreshold,
}
}
Expand Down
6 changes: 5 additions & 1 deletion packages/capacitor-db-sqlite-persistence/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ export type {
CapacitorSQLiteSchemaMismatchPolicy,
SQLiteDBConnection,
} from './capacitor'
export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core'
export {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
persistedCollectionOptions,
} from '@tanstack/db-sqlite-persistence-core'
export type {
PersistedCollectionCoordinator,
PersistedCollectionPersistence,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
SingleProcessCoordinator,
createSQLiteCorePersistenceAdapter,
} from '@tanstack/db-sqlite-persistence-core'
Expand Down Expand Up @@ -79,8 +81,11 @@ function resolveAdapterBaseOptions(
`driver` | `schemaVersion` | `schemaMismatchPolicy`
> {
return {
appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
appliedTxPruneMaxRows:
options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
appliedTxPruneMaxAgeSeconds:
options.appliedTxPruneMaxAgeSeconds ??
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
pullSinceReloadThreshold: options.pullSinceReloadThreshold,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ export type {
CloudflareDOSQLitePersistenceOptions,
DurableObjectStorageLike,
} from './do-persistence'
export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core'
export {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
persistedCollectionOptions,
} from '@tanstack/db-sqlite-persistence-core'
export type {
PersistedCollectionCoordinator,
PersistedCollectionPersistence,
Expand Down
85 changes: 61 additions & 24 deletions packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,21 @@ export type SQLitePullSinceResult<TKey extends string | number> =

const DEFAULT_SCHEMA_VERSION = 1
const DEFAULT_PULL_SINCE_RELOAD_THRESHOLD = 128

/**
* Default cap on retained `applied_tx` rows per collection. The log is a
* replayable cache, so a bounded row count keeps SQLite files from growing
* without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap.
*/
export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS = 1_000

/**
* Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows
* older than this are pruned on the next write. Pass
* `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop.
*/
export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60

const SQLITE_MAX_IN_BATCH_SIZE = 900
const SAFE_IDENTIFIER_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/
const FORBIDDEN_SQL_FRAGMENT_PATTERN = /(;|--|\/\*)/
Expand Down Expand Up @@ -1540,42 +1555,64 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter {
const collectionTableSql = quoteIdentifier(tableMapping.tableName)
const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName)

const [changedRows, deletedRows, latestVersionRows, replayRows] =
await Promise.all([
this.driver.query<{ key: string }>(
`SELECT key
const [
changedRows,
deletedRows,
latestVersionRows,
replayRows,
replayAvailabilityRows,
] = await Promise.all([
this.driver.query<{ key: string }>(
`SELECT key
FROM ${collectionTableSql}
WHERE row_version > ?`,
[fromRowVersion],
),
this.driver.query<{ key: string }>(
`SELECT key
[fromRowVersion],
),
this.driver.query<{ key: string }>(
`SELECT key
FROM ${tombstoneTableSql}
WHERE row_version > ?`,
[fromRowVersion],
),
this.driver.query<{ latest_row_version: number }>(
`SELECT latest_row_version
[fromRowVersion],
),
this.driver.query<{ latest_row_version: number }>(
`SELECT latest_row_version
FROM collection_version
WHERE collection_id = ?
LIMIT 1`,
[collectionId],
),
this.driver.query<{
tx_id: string
row_version: number
replay_json: string | null
replay_requires_full_reload: number
}>(
`SELECT tx_id, row_version, replay_json, replay_requires_full_reload
[collectionId],
),
this.driver.query<{
tx_id: string
row_version: number
replay_json: string | null
replay_requires_full_reload: number
}>(
`SELECT tx_id, row_version, replay_json, replay_requires_full_reload
FROM applied_tx
WHERE collection_id = ? AND row_version > ?
ORDER BY term ASC, seq ASC`,
[collectionId, fromRowVersion],
),
])
[collectionId, fromRowVersion],
),
this.driver.query<{ min_row_version: number | null }>(
`SELECT MIN(row_version) AS min_row_version
FROM applied_tx
WHERE collection_id = ?`,
[collectionId],
),
])

const latestRowVersion = latestVersionRows[0]?.latest_row_version ?? 0
const replayFloor = replayAvailabilityRows[0]?.min_row_version
if (
latestRowVersion > fromRowVersion &&
(replayFloor == null || replayFloor > fromRowVersion + 1)
) {
return {
latestRowVersion,
requiresFullReload: true,
}
}

const changedKeyCount = changedRows.length + deletedRows.length

if (changedKeyCount > this.pullSinceReloadThreshold) {
Expand Down
9 changes: 7 additions & 2 deletions packages/expo-db-sqlite-persistence/src/expo.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
SingleProcessCoordinator,
createSQLiteCorePersistenceAdapter,
} from '@tanstack/db-sqlite-persistence-core'
Expand Down Expand Up @@ -77,8 +79,11 @@ function resolveAdapterBaseOptions(
`driver` | `schemaVersion` | `schemaMismatchPolicy`
> {
return {
appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
appliedTxPruneMaxRows:
options.appliedTxPruneMaxRows ?? DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
appliedTxPruneMaxAgeSeconds:
options.appliedTxPruneMaxAgeSeconds ??
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
pullSinceReloadThreshold: options.pullSinceReloadThreshold,
}
}
Expand Down
6 changes: 5 additions & 1 deletion packages/expo-db-sqlite-persistence/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ export type {
ExpoSQLitePersistenceOptions,
ExpoSQLiteSchemaMismatchPolicy,
} from './expo'
export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core'
export {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
persistedCollectionOptions,
} from '@tanstack/db-sqlite-persistence-core'
export type {
PersistedCollectionCoordinator,
PersistedCollectionPersistence,
Expand Down
29 changes: 29 additions & 0 deletions packages/node-db-sqlite-persistence/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,32 @@ export const todosCollection = createCollection(
mode-specific behavior (`sync-present` vs `sync-absent`) automatically.
- `schemaVersion` is specified per collection via `persistedCollectionOptions`.
- Call `database.close()` when your app shuts down.

## Applied transaction pruning

The `applied_tx` log is a replayable cache, so it is pruned by default to keep
the SQLite file from growing without bound. When you don't pass prune options,
the node driver applies:

- `appliedTxPruneMaxRows: 1_000` (per-collection row cap)
- `appliedTxPruneMaxAgeSeconds: 86_400` (24h age backstop)

Pruning runs inside each write transaction, so every collection self-trims on
its next sync. If a process asks to recover from a point older than the retained
replay window, recovery falls back to a full reload. Override either value to
tune retention, or pass `0` to disable that limit:

```ts
const persistence = createNodeSQLitePersistence({
database,
appliedTxPruneMaxRows: 5_000, // higher row cap
appliedTxPruneMaxAgeSeconds: 0, // disable the age backstop
})
```

Pruning removes rows from `applied_tx`, but SQLite may not immediately shrink
the database file on disk. Use `VACUUM`, `auto_vacuum`, or your own maintenance
process if you need to reclaim disk space.

The defaults are exported as `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` and
`DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS`.
6 changes: 5 additions & 1 deletion packages/node-db-sqlite-persistence/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
export { createNodeSQLitePersistence } from './node-persistence'
export {
createNodeSQLitePersistence,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
} from './node-persistence'
export type {
BetterSqlite3Database,
NodeSQLitePersistenceOptions,
Expand Down
25 changes: 23 additions & 2 deletions packages/node-db-sqlite-persistence/src/node-persistence.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS as CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS as CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
SingleProcessCoordinator,
createSQLiteCorePersistenceAdapter,
} from '@tanstack/db-sqlite-persistence-core'
Expand Down Expand Up @@ -35,6 +37,22 @@ type NodeSQLitePersistenceBaseOptions = Omit<

export type NodeSQLitePersistenceOptions = NodeSQLitePersistenceBaseOptions

/**
* Default cap on retained `applied_tx` rows per collection. The log is a
* replayable cache, so a bounded row count keeps SQLite files from growing
* without limit. Pass `appliedTxPruneMaxRows: 0` to disable the row cap.
*/
export const DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS =
CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS

/**
* Default age backstop for retained `applied_tx` rows, in seconds (24h). Rows
* older than this are pruned on the next write. Pass
* `appliedTxPruneMaxAgeSeconds: 0` to disable the age backstop.
*/
export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS =
CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS

function normalizeSchemaMismatchPolicy(
policy: NodeSQLiteSchemaMismatchPolicy,
): NodeSQLiteCoreSchemaMismatchPolicy {
Expand Down Expand Up @@ -81,8 +99,11 @@ function resolveAdapterBaseOptions(
`driver` | `schemaVersion` | `schemaMismatchPolicy`
> {
return {
appliedTxPruneMaxRows: options.appliedTxPruneMaxRows,
appliedTxPruneMaxAgeSeconds: options.appliedTxPruneMaxAgeSeconds,
appliedTxPruneMaxRows:
options.appliedTxPruneMaxRows ?? CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS,
appliedTxPruneMaxAgeSeconds:
options.appliedTxPruneMaxAgeSeconds ??
CORE_DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS,
pullSinceReloadThreshold: options.pullSinceReloadThreshold,
}
}
Expand Down
Loading
Loading