diff --git a/.oxlintrc.json b/.oxlintrc.json index 6fa2ff8e..03da03f4 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -196,6 +196,12 @@ "threadnote/no-effect-runtime": "off" } }, + { + "files": ["test/helpers/node-*.ts"], + "rules": { + "effecttsgo/node-builtin-import": "off" + } + }, { "files": ["test/**/*.test.ts", "test/**/*.spec.ts", "test/e2e/**/*.e2e.ts"], "rules": { diff --git a/.oxlintrc.max-lines.json b/.oxlintrc.max-lines.json index 828f6267..1f10d754 100644 --- a/.oxlintrc.max-lines.json +++ b/.oxlintrc.max-lines.json @@ -3,19 +3,10 @@ "categories": { "correctness": "off" }, - "ignorePatterns": [ - "src/code_graph/build_status.ts", - "src/code_graph/indexer.ts", - "src/code_graph/inventory.ts", - "src/manager_graph.tsx", - "src/mcp_server.ts", - "src/memory.ts", - "website/src/content/docs.ts" - ], "plugins": [], "rules": { "max-lines": [ - "warn", + "error", { "max": 2000, "skipBlankLines": false, diff --git a/.oxlintrc.strict.json b/.oxlintrc.strict.json index cf012db8..dd7d317d 100644 --- a/.oxlintrc.strict.json +++ b/.oxlintrc.strict.json @@ -70,6 +70,12 @@ "threadnote/no-effect-runtime": "off" } }, + { + "files": ["test/helpers/node-*.ts"], + "rules": { + "effecttsgo/node-builtin-import": "off" + } + }, { "files": ["test/**/*.test.ts", "test/**/*.spec.ts", "test/e2e/**/*.e2e.ts"], "rules": { diff --git a/README.md b/README.md index 0fb9f7a7..82927e1f 100644 --- a/README.md +++ b/README.md @@ -236,11 +236,17 @@ threadnote graph repair --all --dry-run threadnote graph compact --dry-run ``` -`graph status` reports active SQLite database, WAL, and SHM bytes plus page/freelist and reclaimable-byte diagnostics. -When both 512 MiB and 20% of the database are reclaimable it recommends explicit compaction. `graph compact` takes -zero-wait maintenance and checkout locks, verifies the active snapshot before and after SQLite's transactional -`VACUUM`, and defers safely during a build. Preview it with `--dry-run`; `--force` is available below the reviewed -threshold. +`graph status` reports physical SQLite database, WAL, and SHM bytes separately from pages in use and freelist bytes +already reusable inside the database. A large physical file can therefore contain little live graph data without +causing future writes to grow it. When freelist bytes are both at least 512 MiB and 20% of the database, a running +Manager automatically compacts one eligible database at a time after active builders release their locks and sufficient +disk headroom is verified. SQLite can require more than twice the database size as temporary free space during `VACUUM`, so Manager +withholds compaction when that conservative headroom cannot be proved. Automatic compaction runs in an isolated child +process, keeping Manager responsive; Manager shows the latest check, deferral, failure, or reclaimed bytes. Structural +fragmentation analysis can scan live SQLite pages, so it is never scheduled automatically. Compaction rechecks the +active snapshot before and after the transactional rewrite, and interruption leaves the original database intact. +`graph compact --dry-run` remains available to inspect additional fragmentation explicitly, choose the timing, or +troubleshoot; `--force` is the expert override below the reviewed threshold. `graph diagnostics` is home-wide and does not resolve a repository from the current directory. It reports every local graph database, ready snapshot, indexed view, active build, waiter, storage total, health issue, and obsolete store; @@ -260,6 +266,8 @@ before acting. The CLI equivalent for an orphaned store is `threadnote manage` fails fast when graph repair or another native graph maintenance operation is already active; an already-running Manager returns an explicit busy response for graph requests until maintenance finishes. +Manager labels graph views with repository name, the branch observed at a stated boundary, and trusted local folder whenever available; opaque +checkout and worktree identities are reserved for last-resort diagnostics and exact CLI targeting. Maven, Gradle, Kotlin Multiplatform/Android conventions, SwiftPM, conservative Xcode metadata, and nested or integrated Bazel workspaces form a static workspace model; repository build scripts are never executed. Bazel `WORKSPACE`, diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index d70ca312..0a82bf91 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -15,6 +15,7 @@ Direct runtime software and packages bundled into the published JavaScript retai - `remark-gfm` (MIT) - `three` (MIT), used for GPU-accelerated manager graph rendering - `js-yaml` (MIT) +- `yaml` 2.9.0 (ISC), used for comment-preserving Manager Workset manifest updates - TypeScript compiler 5.9 (`typescript-compiler`, Apache-2.0), bundled for native TypeScript/JavaScript graph extraction - `fflate` 0.8.2 (MIT), used for bounded local text extraction from tracked OpenXML, OpenDocument, and EPUB archives - `unpdf` 1.6.2 (MIT) and its bundled PDF.js engine (Apache-2.0), used for local tracked-PDF text and link extraction @@ -42,6 +43,19 @@ Grammar and parser license copies, source revisions, ABIs, and SHA-256 checksums pinned MIT-licensed BGE Small embedding model is installed automatically by `threadnote install`; other model files require an explicit `threadnote models install` action. Catalog entries identify every model source and license. +### `yaml` 2.9.0 license notice + +Copyright Eemeli Aro + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby +granted, provided that the above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN +AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + ## Public website The separately deployed GitHub Pages website is not part of the standalone release. Its build uses Vite and diff --git a/bun.lock b/bun.lock index f378e56e..5c7e9865 100644 --- a/bun.lock +++ b/bun.lock @@ -14,6 +14,7 @@ "typescript-compiler": "npm:typescript@5.9.3", "unpdf": "1.6.2", "web-tree-sitter": "0.26.11", + "yaml": "^2.9.0", }, "devDependencies": { "@effect/ai-openai-compat": "4.0.0-beta.102", diff --git a/manager/app.css b/manager/app.css index c033c71a..814806e7 100644 --- a/manager/app.css +++ b/manager/app.css @@ -2714,6 +2714,465 @@ dd { color: var(--danger); } +.worksets-workspace { + display: grid; + gap: 14px; + grid-template-columns: 270px minmax(0, 1fr); + height: 100%; + min-height: 0; +} + +.worksets-catalog, +.worksets-main, +.worksets-card { + background: var(--panel); + border: 1px solid var(--line); + border-radius: var(--radius); +} + +.worksets-catalog { + display: flex; + flex-direction: column; + min-height: 0; + padding: 14px; +} + +.worksets-main { + min-width: 0; + overflow: auto; + padding: 18px; +} + +.worksets-section-head, +.worksets-header, +.worksets-editor header, +.worksets-editor footer { + align-items: center; + display: flex; + gap: 12px; + justify-content: space-between; +} + +.worksets-section-head h2, +.worksets-section-head h3, +.worksets-header h2, +.worksets-editor h2 { + margin: 0; +} + +.worksets-header { + align-items: flex-start; + margin-bottom: 14px; +} + +.worksets-header > div:first-child { + min-width: 0; +} + +.worksets-header p:not(.eyebrow), +.worksets-boundary, +.worksets-muted, +.worksets-definition-list p { + color: var(--muted); + font-size: 12px; + line-height: 1.5; +} + +.worksets-boundary { + border-bottom: 1px solid var(--line-soft); + margin: 10px 0; + padding-bottom: 10px; +} + +.worksets-definition-list { + display: grid; + gap: 6px; + margin-bottom: 12px; + min-height: 0; + overflow: auto; +} + +.worksets-definition-list button { + display: grid; + gap: 4px; + justify-items: start; + text-align: left; +} + +.worksets-definition-list button.is-selected { + background: var(--accent-soft); + border-color: var(--accent-line); +} + +.worksets-definition-list span { + color: var(--muted); + font-size: 11px; +} + +.worksets-catalog > .quiet-button { + margin-top: auto; +} + +.worksets-card { + margin-top: 14px; + padding: 16px; +} + +.worksets-notice, +.worksets-warning, +.worksets-error { + border-radius: 8px; + font-size: 12px; + line-height: 1.45; + margin: 9px 0; + padding: 8px 10px; +} + +.worksets-notice { + background: var(--accent-soft); + color: var(--accent); +} + +.worksets-warning { + background: var(--warn-soft); + color: var(--warn); +} + +.worksets-error { + background: var(--danger-soft); + color: var(--danger); +} + +.worksets-metrics { + display: grid; + gap: 8px; + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin: 14px 0; +} + +.worksets-metrics > div { + background: var(--panel-alt); + border: 1px solid var(--line-soft); + border-radius: 9px; + display: grid; + gap: 5px; + min-width: 0; + padding: 10px; +} + +.worksets-metrics span, +.worksets-metrics strong { + overflow-wrap: anywhere; +} + +.worksets-metrics span { + color: var(--muted); + font-size: 10px; +} + +.worksets-member-grid { + display: grid; + gap: 8px; + grid-template-columns: repeat(auto-fill, minmax(190px, 1fr)); +} + +.worksets-member-grid article, +.worksets-evidence-list article, +.worksets-edge-list article, +.worksets-topology-grid article { + background: var(--panel-alt); + border: 1px solid var(--line-soft); + border-radius: 9px; + display: grid; + gap: 5px; + min-width: 0; + padding: 10px; +} + +.worksets-member-grid small, +.worksets-evidence-list small, +.worksets-edge-list small, +.worksets-topology-grid small, +.worksets-receipt { + color: var(--muted); + font-size: 10px; + line-height: 1.45; + overflow-wrap: anywhere; +} + +.worksets-state { + border-radius: 999px; + font-size: 10px; + justify-self: start; + padding: 2px 7px; +} + +.worksets-state.is-ok { + background: var(--ok-soft); + color: var(--ok); +} + +.worksets-state.is-warn { + background: var(--warn-soft); + color: var(--warn); +} + +.worksets-state.is-fail { + background: var(--danger-soft); + color: var(--danger); +} + +.worksets-job { + align-items: center; + background: var(--panel-alt); + border: 1px solid var(--accent-line); + border-radius: var(--radius); + display: flex; + flex-wrap: wrap; + gap: 10px 16px; + margin-top: 14px; + padding: 12px; +} + +.worksets-job > div { + align-items: center; + display: flex; + gap: 8px; + min-width: min(100%, 340px); +} + +.worksets-job > span { + color: var(--muted); + font-size: 11px; + margin-left: auto; +} + +.worksets-job-receipts { + display: grid !important; + flex-basis: 100%; + gap: 8px !important; + grid-template-columns: repeat(auto-fill, minmax(190px, 1fr)); +} + +.worksets-job-receipts article { + background: var(--panel); + border: 1px solid var(--line-soft); + border-radius: 9px; + display: grid; + gap: 5px; + min-width: 0; + padding: 9px; +} + +.worksets-job-receipts small { + color: var(--muted); + font-size: 10px; + overflow-wrap: anywhere; +} + +.worksets-operation-tabs { + border-bottom: 1px solid var(--line); + display: flex; + gap: 4px; + margin-top: 18px; + overflow-x: auto; +} + +.worksets-operation-tabs button { + background: transparent; + border-color: transparent; + border-radius: 8px 8px 0 0; +} + +.worksets-operation-tabs button.is-active { + background: var(--accent-soft); + color: var(--accent); +} + +.worksets-operation { + display: grid; + gap: 12px; + margin-top: 0; +} + +.worksets-form-row, +.worksets-topology-grid { + display: grid; + gap: 12px; + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.worksets-operation label, +.worksets-editor > label { + display: grid; + font-size: 12px; + font-weight: 650; + gap: 6px; +} + +.worksets-checks { + display: flex; + flex-wrap: wrap; + gap: 14px; +} + +.worksets-checks label, +.worksets-project-picker label { + align-items: center; + display: flex; + font-size: 12px; + gap: 7px; +} + +.worksets-checks input, +.worksets-project-picker input { + min-height: auto; + width: auto; +} + +.worksets-evidence-list, +.worksets-edge-list, +.worksets-topology-grid > div { + display: grid; + gap: 8px; +} + +.worksets-evidence-list article header { + align-items: flex-start; + display: flex; + gap: 8px; + justify-content: space-between; +} + +.worksets-evidence-list article header span, +.worksets-edge-list article > span, +.worksets-topology-grid article > span { + color: var(--accent); + font-size: 11px; + overflow-wrap: anywhere; +} + +.worksets-evidence-list code { + color: var(--muted); + font-size: 11px; + overflow-wrap: anywhere; +} + +.worksets-reference-actions { + border-top: 1px solid var(--line-soft); + color: var(--muted); + font-size: 11px; + padding-top: 7px; +} + +.worksets-reference-actions > code { + display: block; + margin: 7px 0; +} + +.worksets-evidence-list p, +.worksets-brief-section p { + line-height: 1.45; + margin: 0; +} + +.worksets-topology-grid h4, +.worksets-brief-section h4 { + font-size: 13px; + margin: 8px 0 0; +} + +.worksets-brief-section { + border-top: 1px solid var(--line-soft); + display: grid; + gap: 7px; + padding-top: 10px; +} + +.worksets-brief-section p { + background: var(--panel-alt); + border-radius: 7px; + font-size: 12px; + padding: 8px; +} + +.worksets-empty { + align-items: center; + display: flex; + flex-direction: column; + height: 100%; + justify-content: center; + text-align: center; +} + +.worksets-empty p { + color: var(--muted); +} + +.worksets-editor-backdrop { + align-items: center; + background: rgba(3, 6, 10, 0.76); + display: flex; + inset: 0; + justify-content: center; + padding: 20px; + position: fixed; + z-index: 30; +} + +.worksets-editor { + background: var(--panel); + border: 1px solid var(--accent-line); + border-radius: 15px; + box-shadow: var(--shadow); + display: grid; + gap: 14px; + max-height: min(760px, calc(100dvh - 40px)); + overflow: hidden; + padding: 18px; + width: min(620px, 100%); +} + +.worksets-project-picker { + border: 1px solid var(--line); + border-radius: 8px; + display: grid; + gap: 8px; + max-height: 280px; + overflow: auto; + padding: 10px; +} + +.worksets-project-picker-controls { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.worksets-project-picker-controls span { + color: var(--muted); + font-size: 11px; + margin-right: auto; +} + +.worksets-project-picker label > span { + display: grid; + gap: 3px; + min-width: 0; +} + +.worksets-project-picker small { + color: var(--muted); + overflow-wrap: anywhere; +} + +.worksets-editor footer { + border-top: 1px solid var(--line-soft); + justify-content: flex-end; + padding-top: 12px; +} + .sr-only { border: 0; clip: rect(0, 0, 0, 0); @@ -2727,6 +3186,14 @@ dd { } @media (max-width: 1180px) { + .worksets-workspace { + grid-template-columns: 230px minmax(0, 1fr); + } + + .worksets-metrics { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .graph-toolbar { grid-template-areas: 'scope scope' @@ -2776,7 +3243,7 @@ dd { } .primary-nav { - grid-template-columns: repeat(5, minmax(0, 1fr)); + grid-template-columns: repeat(6, minmax(0, 1fr)); } .primary-nav > button { @@ -2803,6 +3270,29 @@ dd { min-height: 760px; } + .worksets-workspace { + grid-template-columns: 1fr; + height: auto; + } + + .worksets-catalog { + max-height: 340px; + } + + .worksets-main { + min-height: 620px; + overflow: visible; + } + + .worksets-header { + align-items: stretch; + flex-direction: column; + } + + .worksets-header .button-row { + justify-content: flex-start; + } + .graph-workspace { min-height: 690px; padding: 14px; @@ -2856,6 +3346,47 @@ dd { flex-direction: column; } + .primary-nav { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + + .worksets-main, + .worksets-catalog { + padding: 12px; + } + + .worksets-form-row, + .worksets-metrics, + .worksets-topology-grid { + grid-template-columns: 1fr; + } + + .worksets-section-head, + .worksets-editor header, + .worksets-editor footer { + align-items: stretch; + flex-wrap: wrap; + } + + .worksets-operation-tabs button { + min-width: max-content; + } + + .worksets-editor-backdrop { + align-items: stretch; + padding: 8px; + } + + .worksets-editor { + max-height: calc(100dvh - 16px); + overflow-y: auto; + padding: 14px; + } + + .worksets-editor footer button { + flex: 1 1 130px; + } + .graph-toolbar { grid-template-areas: 'scope' diff --git a/package.json b/package.json index a2e70cbd..d8beb50a 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dev:mcp-server": "bun src/standalone.ts mcp-server", "lint": "bun scripts/lint.ts && bun run lint:file-length", "lint:file-length": "bun scripts/lint-file-length.ts", - "lint:fix": "bun --bun oxlint --fix config/lint src test scripts website/src website/vite.config.ts && bun run lint:file-length", + "lint:fix": "bun --bun oxlint --config .oxlintrc.strict.json --threads=1 --deny-warnings --report-unused-disable-directives-severity=error --ignore-pattern 'test/evaluation/fixtures/**/repository/**' --fix config/lint src test scripts website/src website/vite.config.ts && bun run lint:file-length", "prepare": "bun --bun husky && bun --bun effect-tsgo patch --no-typescript --oxlint", "precommit": "bun run lint && bun run prettier:write && bun run typecheck", "prettier:check": "bun --bun prettier --check \"**/*.{ts,tsx,js,cjs,json,md,yaml,yml,css,html}\"", @@ -73,7 +73,7 @@ "site:typecheck": "bun --bun tsc -p website/tsconfig.json --noEmit", "site:test": "bun --bun vitest run test/unit/website-content.test.ts test/unit/website-release-boundary.test.ts", "site:bind-performance-evidence": "bun scripts/site-performance-evidence.ts", - "site:check": "bun run site:typecheck && bun --bun oxlint website/src website/vite.config.ts && bun scripts/lint-file-length.ts website/src && bun run site:test", + "site:check": "bun run site:typecheck && bun --bun oxlint --config .oxlintrc.strict.json --threads=1 --deny-warnings --report-unused-disable-directives-severity=error website/src website/vite.config.ts && bun scripts/lint-file-length.ts website/src && bun run site:test", "train:reranker:data": "bun scripts/build-recall-reranker-dataset.ts", "train:reranker:prepare": "bun scripts/training/prepare-reviewed-recall-reranker-dataset.ts", "train:reranker:validate": "bun scripts/validate-recall-reranker-dataset.ts", @@ -129,7 +129,8 @@ "three": "^0.185.1", "typescript-compiler": "npm:typescript@5.9.3", "unpdf": "1.6.2", - "web-tree-sitter": "0.26.11" + "web-tree-sitter": "0.26.11", + "yaml": "^2.9.0" }, "overrides": { "@effect/platform-node-shared": "4.0.0-beta.102", diff --git a/scripts/archive-release.ts b/scripts/archive-release.ts index 68bcf433..e2e0ac9c 100644 --- a/scripts/archive-release.ts +++ b/scripts/archive-release.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Layer, Path} from 'effect'; @@ -15,7 +16,7 @@ const archiveRelease = Effect.gen(function* () { const target = Bun.env.THREADNOTE_RELEASE_TARGET?.trim(); if (!target || !ARCHIVE_TARGET_PATTERN.test(target)) { return yield* Effect.fail( - new Error('THREADNOTE_RELEASE_TARGET must be one of darwin|linux|windows combined with arm64|x64.'), + new ScriptError('THREADNOTE_RELEASE_TARGET must be one of darwin|linux|windows combined with arm64|x64.'), ); } @@ -27,7 +28,7 @@ const archiveRelease = Effect.gen(function* () { const artifactPath = path.join(artifactsRoot, artifactName); const checksumPath = `${artifactPath}.sha256`; if (!(yield* fs.exists(path.join(distributionRoot, 'release.json')))) { - return yield* Effect.fail(new Error('dist/release.json is missing; build the release before archiving it.')); + return yield* Effect.fail(new ScriptError('dist/release.json is missing; build the release before archiving it.')); } yield* fs.makeDirectory(artifactsRoot, {recursive: true}); @@ -45,4 +46,4 @@ const systemLayer = SystemInfo.layer; const commandLayer = CommandExecutor.layer.pipe(Layer.provide(systemLayer)); const archiveLayer = Layer.merge(systemLayer, commandLayer).pipe(Layer.provideMerge(BunServices.layer)); -BunRuntime.runMain(archiveRelease.pipe(Effect.provide(archiveLayer))); +BunRuntime.runMain(provideScriptLayer(archiveRelease, archiveLayer)); diff --git a/scripts/benchmark-code-graph-dirty-overlay.ts b/scripts/benchmark-code-graph-dirty-overlay.ts index a4274441..996b1143 100644 --- a/scripts/benchmark-code-graph-dirty-overlay.ts +++ b/scripts/benchmark-code-graph-dirty-overlay.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Clock, Effect, FileSystem, Path} from 'effect'; import {CodeGraphIndexer, type CodeGraphIndexerShape} from '../src/code_graph/indexer.js'; @@ -31,7 +32,7 @@ const benchmarkCodeGraphDirtyOverlay = Effect.scoped( Effect.gen(function* () { const options = parseArguments(yield* scriptArguments()); const system = yield* SystemInfo; - const hardware = yield* system.hardwareInfo(); + const hardware = yield* system.hardwareInfo; const indexer = yield* CodeGraphIndexer; const incremental: DirtyOverlayObservation[] = []; const full: DirtyOverlayObservation[] = []; @@ -52,7 +53,9 @@ const benchmarkCodeGraphDirtyOverlay = Effect.scoped( incrementalSample.edges !== fullSample.edges || incrementalSample.totalFiles !== fullSample.totalFiles ) { - return yield* Effect.fail(new Error(`Dirty-overlay benchmark graph shape diverged in sample ${index + 1}.`)); + return yield* Effect.fail( + new ScriptError(`Dirty-overlay benchmark graph shape diverged in sample ${index + 1}.`), + ); } } @@ -111,7 +114,7 @@ const runDirtyOverlayIndex = Effect.fn('benchmarkCodeGraphDirtyOverlay.run')(fun const changedPath = path.join(prepared.repository, 'src/module-00000.ts'); const committed = yield* fs.readFileString(changedPath); if (!committed.includes('return 0;')) { - return yield* Effect.fail(new Error('Dirty-overlay benchmark fixture lost its body-only edit marker.')); + return yield* Effect.fail(new ScriptError('Dirty-overlay benchmark fixture lost its body-only edit marker.')); } yield* fs.writeFileString(changedPath, committed.replace('return 0;', 'return 1000000;')); @@ -158,12 +161,14 @@ const runDirtyOverlayIndex = Effect.fn('benchmarkCodeGraphDirtyOverlay.run')(fun function validateMaterialization(summary: CodeGraphIndexSummary, incrementalOverlay: boolean): void { if (incrementalOverlay) { if (summary.materialization?.mode !== 'incremental-overlay' || summary.materialization.stagedFiles !== 1) { - throw new Error(`Incremental dirty-overlay benchmark fell back: ${JSON.stringify(summary.materialization)}.`); + throw new ScriptError( + `Incremental dirty-overlay benchmark fell back: ${JSON.stringify(summary.materialization)}.`, + ); } return; } if (summary.materialization?.mode !== 'full' || summary.materialization.fallbackReason !== 'disabled') { - throw new Error( + throw new ScriptError( `Full dirty-overlay benchmark did not use its control path: ${JSON.stringify(summary.materialization)}.`, ); } @@ -174,7 +179,7 @@ function summarize(values: readonly number[]): { readonly mean: number; readonly minimum: number; } { - if (values.length === 0) throw new Error('Dirty-overlay benchmark requires at least one observation.'); + if (values.length === 0) throw new ScriptError('Dirty-overlay benchmark requires at least one observation.'); return { maximum: Math.max(...values), mean: values.reduce((total, value) => total + value, 0) / values.length, @@ -195,20 +200,20 @@ function parseArguments(args: readonly string[]): DirtyOverlayBenchmarkOptions { if (argument === '--output') outputPath = required(args[++index], argument); else if (argument === '--samples') samples = integer(args[++index], argument, 1); else if (argument === '--scale-symbols') scaleSymbols = integer(args[++index], argument, 1); - else throw new Error(`Unknown dirty-overlay benchmark option: ${argument}`); + else throw new ScriptError(`Unknown dirty-overlay benchmark option: ${argument}`); } return {outputPath, samples, scaleSymbols}; } function integer(value: string | undefined, option: string, minimum: number): number { const parsed = Number.parseInt(required(value, option), 10); - if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`${option} must be at least ${minimum}`); + if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new ScriptError(`${option} must be at least ${minimum}`); return parsed; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } -BunRuntime.runMain(benchmarkCodeGraphDirtyOverlay.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(benchmarkCodeGraphDirtyOverlay, ApplicationLayer)); diff --git a/scripts/benchmark-code-graph-heavy-tail.ts b/scripts/benchmark-code-graph-heavy-tail.ts index 67457a63..26186aa9 100644 --- a/scripts/benchmark-code-graph-heavy-tail.ts +++ b/scripts/benchmark-code-graph-heavy-tail.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Database} from 'bun:sqlite'; import {Effect, Exit, FileSystem, Path} from 'effect'; @@ -175,19 +176,19 @@ const runParent = Effect.fn('benchmarkCodeGraphHeavyTail.parent')(function* (arg validateCompletedRun('parallel-worker', parallel, profile); validateCompletedRun('resumed', resumed, profile); if (interrupted.state !== 'interrupted' || interrupted.cache.files < 1) { - return yield* Effect.fail(new Error('The interruption run did not retain any durable parser cache rows.')); + return yield* Effect.fail(new ScriptError('The interruption run did not retain any durable parser cache rows.')); } if ((resumed.reusedFiles ?? 0) < 1) { - return yield* Effect.fail(new Error('The resumed run did not reuse facts persisted before interruption.')); + return yield* Effect.fail(new ScriptError('The resumed run did not reuse facts persisted before interruption.')); } if (single.graph!.digest !== parallel.graph!.digest) { - return yield* Effect.fail(new Error('Single-worker and parallel code graphs differ.')); + return yield* Effect.fail(new ScriptError('Single-worker and parallel code graphs differ.')); } if (single.graph!.digest !== resumed.graph!.digest) { - return yield* Effect.fail(new Error('Interrupted/resumed and clean code graphs differ.')); + return yield* Effect.fail(new ScriptError('Interrupted/resumed and clean code graphs differ.')); } - const hardware = yield* system.hardwareInfo(); + const hardware = yield* system.hardwareInfo; const [commit, dirty] = yield* Effect.all( [git(process.cwd(), ['rev-parse', 'HEAD']), git(process.cwd(), ['status', '--porcelain'])], {concurrency: 2}, @@ -261,7 +262,7 @@ const runChild = Effect.fn('benchmarkCodeGraphHeavyTail.child')(function* (args: return false; }).pipe( Effect.flatMap(shouldInterrupt => - shouldInterrupt ? Effect.fail(new Error('Expected heavy-tail benchmark interruption.')) : Effect.void, + shouldInterrupt ? Effect.fail(new ScriptError('Expected heavy-tail benchmark interruption.')) : Effect.void, ), ), threadnoteHome: home, @@ -291,7 +292,7 @@ const runChild = Effect.fn('benchmarkCodeGraphHeavyTail.child')(function* (args: return; } if (args.interruptAfterPersistedFiles !== undefined) { - return yield* Effect.fail(new Error('The heavy-tail benchmark completed before its requested interruption.')); + return yield* Effect.fail(new ScriptError('The heavy-tail benchmark completed before its requested interruption.')); } const summary = exit.value; const graph = yield* store.loadGraph(layout.databasePath, summary.snapshot.id); @@ -400,24 +401,24 @@ function heavyTailGraphShape(graph: StoredCodeGraph) { } function validateCompletedRun(name: string, run: HeavyTailChildRun, profile: CodeGraphHeavyTailProfile): void { - if (run.state !== 'complete' || !run.graph) throw new Error(`${name} heavy-tail run did not complete.`); + if (run.state !== 'complete' || !run.graph) throw new ScriptError(`${name} heavy-tail run did not complete.`); if (run.graph.lowSignalJsonSymbols !== 0 || run.cache.lowSignalJsonFactsBytes !== 0) { - throw new Error(`${name} heavy-tail run admitted excluded low-signal JSON.`); + throw new ScriptError(`${name} heavy-tail run admitted excluded low-signal JSON.`); } if (run.graph.pathologicalTypeScriptTails !== profile.pathologicalTypeScriptFiles) { - throw new Error(`${name} heavy-tail run lost declarations after pathological TypeScript calls.`); + throw new ScriptError(`${name} heavy-tail run lost declarations after pathological TypeScript calls.`); } if (!run.graph.generatedTypeScriptTailPreserved) { - throw new Error(`${name} heavy-tail run lost declarations from generated TypeScript surface extraction.`); + throw new ScriptError(`${name} heavy-tail run lost declarations from generated TypeScript surface extraction.`); } if (run.graph.textlessSvgSymbols !== 0) { - throw new Error(`${name} heavy-tail run admitted excluded textless SVG.`); + throw new ScriptError(`${name} heavy-tail run admitted excluded textless SVG.`); } if (run.graph.files !== codeGraphHeavyTailEligibleFiles(profile)) { - throw new Error(`${name} heavy-tail run indexed ${run.graph.files} files; expected fixture shape mismatch.`); + throw new ScriptError(`${name} heavy-tail run indexed ${run.graph.files} files; expected fixture shape mismatch.`); } if (Object.values(run.languages).some(language => language.degradedFiles > 0)) { - throw new Error(`${name} heavy-tail run degraded one or more parser files.`); + throw new ScriptError(`${name} heavy-tail run degraded one or more parser files.`); } } @@ -483,7 +484,7 @@ const spawnChild = Effect.fn('benchmarkCodeGraphHeavyTail.spawnChild')(function* ); if (exitCode !== 0) { return yield* Effect.fail( - new Error( + new ScriptError( `${options.name} heavy-tail child exited with ${exitCode}.\n` + boundedOutput('stdout', stdout) + boundedOutput('stderr', stderr), @@ -494,7 +495,8 @@ const spawnChild = Effect.fn('benchmarkCodeGraphHeavyTail.spawnChild')(function* }); export function parseHeavyTailChildRun(value: unknown): HeavyTailChildRun { - if (typeof value !== 'object' || value === null) throw new Error('Heavy-tail child artifact must be an object.'); + if (typeof value !== 'object' || value === null) + throw new ScriptError('Heavy-tail child artifact must be an object.'); const artifact = value as Partial; if ( artifact.version !== 1 || @@ -513,19 +515,20 @@ export function parseHeavyTailChildRun(value: unknown): HeavyTailChildRun { artifact.languages === null || !Array.isArray(artifact.slowFiles) ) { - throw new Error('Heavy-tail child artifact is invalid.'); + throw new ScriptError('Heavy-tail child artifact is invalid.'); } if (artifact.state === 'complete' && artifact.graph === undefined) { - throw new Error('Completed heavy-tail child artifact must include a graph shape.'); + throw new ScriptError('Completed heavy-tail child artifact must include a graph shape.'); } if (artifact.state === 'interrupted' && !positiveInteger(artifact.interruptedAfterPersistedFiles)) { - throw new Error('Interrupted heavy-tail child artifact must include its durable interruption point.'); + throw new ScriptError('Interrupted heavy-tail child artifact must include its durable interruption point.'); } return artifact as HeavyTailChildRun; } export function parseCodeGraphHeavyTailBenchmarkArtifact(value: unknown): CodeGraphHeavyTailBenchmarkArtifact { - if (typeof value !== 'object' || value === null) throw new Error('Heavy-tail benchmark artifact must be an object.'); + if (typeof value !== 'object' || value === null) + throw new ScriptError('Heavy-tail benchmark artifact must be an object.'); const artifact = value as Partial; if ( artifact.version !== 1 || @@ -535,7 +538,7 @@ export function parseCodeGraphHeavyTailBenchmarkArtifact(value: unknown): CodeGr typeof artifact.runs !== 'object' || artifact.runs === null ) { - throw new Error('Heavy-tail benchmark artifact is invalid.'); + throw new ScriptError('Heavy-tail benchmark artifact is invalid.'); } parseCodeGraphHeavyTailProfile(artifact.profile); parseHeavyTailChildRun(artifact.runs.single); @@ -565,15 +568,15 @@ function parseArguments(args: readonly string[]): BenchmarkArguments { else if (argument === '--repository') repository = required(args[++index], argument); else if (argument === '--smoke') smoke = true; else if (argument === '--workers') workers = integer(args[++index], argument, 1, 8); - else throw new Error(`Unknown heavy-tail benchmark option: ${argument}`); + else throw new ScriptError(`Unknown heavy-tail benchmark option: ${argument}`); } if ( !child && [home, profilePath, repository, workers, interruptAfterPersistedFiles].some(value => value !== undefined) ) { - throw new Error('Child-only heavy-tail benchmark options require --child.'); + throw new ScriptError('Child-only heavy-tail benchmark options require --child.'); } - if (child && smoke) throw new Error('--smoke is a parent benchmark option.'); + if (child && smoke) throw new ScriptError('--smoke is a parent benchmark option.'); return {child, home, interruptAfterPersistedFiles, outputPath, profilePath, repository, smoke, workers}; } @@ -585,13 +588,13 @@ function integer( ): number { const parsed = Number.parseInt(required(value, option), 10); if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new Error(`${option} must be between ${minimum} and ${maximum}.`); + throw new ScriptError(`${option} must be between ${minimum} and ${maximum}.`); } return parsed; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } @@ -628,4 +631,4 @@ const git = Effect.fn('benchmarkCodeGraphHeavyTail.git')((cwd: string, args: rea ), ); -if (import.meta.main) BunRuntime.runMain(benchmark.pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(benchmark, ApplicationLayer)); diff --git a/scripts/benchmark-code-graph-lexical-production-arguments.ts b/scripts/benchmark-code-graph-lexical-production-arguments.ts index 4fdf7266..16259bed 100644 --- a/scripts/benchmark-code-graph-lexical-production-arguments.ts +++ b/scripts/benchmark-code-graph-lexical-production-arguments.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {Option} from 'effect'; const DEFAULT_SYMBOL_COUNT = 5_000; @@ -21,14 +22,14 @@ export function parseLexicalProductionBenchmarkArguments( const symbolCount = integerArgument(arguments_, '--symbols', DEFAULT_SYMBOL_COUNT); const allowLarge = arguments_.includes('--allow-large'); if (symbolCount > MAXIMUM_SYMBOLS || (symbolCount > MAXIMUM_SYMBOLS_WITHOUT_LARGE_OPT_IN && !allowLarge)) { - throw new Error( + throw new ScriptError( `--symbols must be at most ${MAXIMUM_SYMBOLS_WITHOUT_LARGE_OPT_IN} without --allow-large and ${MAXIMUM_SYMBOLS} overall.`, ); } let outputPath = Option.none(); if (outputIndex !== -1) { const value = arguments_[outputIndex + 1]; - if (!value) throw new Error('--output requires a path.'); + if (!value) throw new ScriptError('--output requires a path.'); outputPath = Option.some(value); } return { @@ -44,6 +45,6 @@ function integerArgument(arguments_: readonly string[], name: string, fallback: const index = arguments_.indexOf(name); if (index === -1) return fallback; const value = Number(arguments_[index + 1]); - if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} requires a positive integer.`); + if (!Number.isSafeInteger(value) || value <= 0) throw new ScriptError(`${name} requires a positive integer.`); return value; } diff --git a/scripts/benchmark-code-graph-lexical-production.ts b/scripts/benchmark-code-graph-lexical-production.ts index b3768f4f..015602de 100644 --- a/scripts/benchmark-code-graph-lexical-production.ts +++ b/scripts/benchmark-code-graph-lexical-production.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Database} from 'bun:sqlite'; import {Effect, FileSystem, Option, Path} from 'effect'; @@ -70,7 +71,7 @@ const benchmark = Effect.scoped( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const system = yield* SystemInfo; - const hardware = yield* system.hardwareInfo(); + const hardware = yield* system.hardwareInfo; const root = yield* fs.makeTempDirectoryScoped({prefix: 'threadnote-lexical-production-micro-'}); const compactPath = path.join(root, 'compact.sqlite'); const legacyPath = path.join(root, 'legacy.sqlite'); @@ -111,7 +112,7 @@ const benchmark = Effect.scoped( }; if (Object.values(assertions).some(value => !value)) { return yield* Effect.fail( - new Error(`Code graph lexical production microbenchmark failed: ${JSON.stringify(assertions)}`), + new ScriptError(`Code graph lexical production microbenchmark failed: ${JSON.stringify(assertions)}`), ); } @@ -360,7 +361,7 @@ function compactValidationMeasurements(databasePath: string, snapshotId: string) result = query.get(...statement.parameters) as typeof result; durations.push(performance.now() - startedAt); } - if (result === undefined) throw new Error('Compact lexical deep audit did not return a storage receipt.'); + if (result === undefined) throw new ScriptError('Compact lexical deep audit did not return a storage receipt.'); return { actualPostingCount: Number(result.posting_count), actualSymbolCount: Number(result.symbol_count), @@ -461,7 +462,7 @@ function storageMeasurements(databasePath: string): StorageMeasurements { function pragmaInteger(database: Database, name: 'freelist_count' | 'page_count' | 'page_size'): number { const row = database.query(`PRAGMA ${name}`).get() as Record | null; const value = Number(row?.[name] ?? -1); - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`SQLite returned an invalid ${name}.`); + if (!Number.isSafeInteger(value) || value < 0) throw new ScriptError(`SQLite returned an invalid ${name}.`); return value; } @@ -548,4 +549,4 @@ function gitValue(arguments_: readonly string[]): string { return result.exitCode === 0 ? new TextDecoder().decode(result.stdout).trim() : 'unknown'; } -if (import.meta.main) benchmark.pipe(Effect.provide(ApplicationLayer), BunRuntime.runMain); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(benchmark, ApplicationLayer)); diff --git a/scripts/benchmark-code-graph-workset.ts b/scripts/benchmark-code-graph-workset.ts index 8f170718..185ba0f1 100644 --- a/scripts/benchmark-code-graph-workset.ts +++ b/scripts/benchmark-code-graph-workset.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; @@ -50,7 +51,7 @@ export function parseCodeGraphWorksetBenchmarkArguments(args: readonly string[]) else if (argument === '--samples') samples = integer(args[++index], argument, 1, 100); else if (argument === '--sizes') sizes = parseBenchmarkSizes(required(args[++index], argument)); else if (argument === '--warmups') warmups = integer(args[++index], argument, 0, 100); - else throw new Error(`Unknown code graph workset benchmark option: ${argument}`); + else throw new ScriptError(`Unknown code graph workset benchmark option: ${argument}`); } return {failOnBudget, outputPath, samples, sizes, warmups}; } @@ -62,29 +63,29 @@ export const benchmarkCodeGraphWorkset = Effect.scoped( const prepared = yield* Effect.acquireRelease( Effect.tryPromise({ try: () => prepareCodeGraphWorksetFixture({size: maximumSize, stateProfile: 'all-clean'}), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }), fixture => Effect.tryPromise({ try: () => removePreparedCodeGraphWorksetFixture(fixture), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }).pipe(Effect.catch(() => Effect.void)), ); yield* indexPreparedCodeGraphWorksetFixture(prepared); const selectedWorksets = options.sizes.map(size => { const workset = prepared.plan.worksets.find(candidate => candidate.size === size); - if (!workset) throw new Error(`Fixture did not emit a size-${size} workset.`); + if (!workset) throw new ScriptError(`Fixture did not emit a size-${size} workset.`); return workset.name; }); yield* publishIndexedCodeGraphWorksetCatalog(prepared, selectedWorksets); const config = codeGraphWorksetRuntimeConfig(prepared); const query = prepared.plan.queries.find(candidate => candidate.id === BENCHMARK_QUERY_ID); - if (!query) return yield* Effect.fail(new Error(`Fixture is missing benchmark query ${BENCHMARK_QUERY_ID}.`)); + if (!query) return yield* Effect.fail(new ScriptError(`Fixture is missing benchmark query ${BENCHMARK_QUERY_ID}.`)); const samples = []; for (const worksetSize of options.sizes) { const workset = prepared.plan.worksets.find(candidate => candidate.size === worksetSize); - if (!workset) return yield* Effect.fail(new Error(`Fixture did not emit a size-${worksetSize} workset.`)); + if (!workset) return yield* Effect.fail(new ScriptError(`Fixture did not emit a size-${worksetSize} workset.`)); for (let warmup = 0; warmup < options.warmups; warmup += 1) { yield* measureCodeGraphWorksetQuery(config, workset.name, query.query); } @@ -96,7 +97,7 @@ export const benchmarkCodeGraphWorkset = Effect.scoped( const measurements = codeGraphWorksetBenchmarkMeasurements(samples); const system = yield* SystemInfo; - const hardware = yield* system.hardwareInfo(); + const hardware = yield* system.hardwareInfo; const [commit, dirty] = yield* Effect.all( [sourceGit(['rev-parse', 'HEAD']), sourceGit(['status', '--porcelain'])], { @@ -137,7 +138,7 @@ export const benchmarkCodeGraphWorkset = Effect.scoped( yield* printJson(artifact); if (options.failOnBudget) { const failures = codeGraphWorksetBenchmarkBudgetFailures(measurements, options.sizes); - if (failures.length > 0) return yield* Effect.fail(new Error(failures.join('\n'))); + if (failures.length > 0) return yield* Effect.fail(new ScriptError(failures.join('\n'))); } }), ); @@ -145,16 +146,16 @@ export const benchmarkCodeGraphWorkset = Effect.scoped( function parseBenchmarkSizes(value: string): readonly CodeGraphWorksetFixtureSize[] { const parts = value.split(','); if (parts.length === 0 || parts.some(part => !part.trim())) - throw new Error('--sizes requires comma-separated sizes.'); + throw new ScriptError('--sizes requires comma-separated sizes.'); const sizes = parts.map(part => Number(part.trim())); if (sizes.some(size => !Number.isSafeInteger(size) || size < 1)) { - throw new Error('--sizes requires positive integer sizes.'); + throw new ScriptError('--sizes requires positive integer sizes.'); } - if (new Set(sizes).size !== sizes.length) throw new Error('--sizes sizes must be unique.'); + if (new Set(sizes).size !== sizes.length) throw new ScriptError('--sizes sizes must be unique.'); const allowed = new Set(CODE_GRAPH_WORKSET_FIXTURE_SUPPORTED_SIZES); for (const size of sizes) { if (!allowed.has(size)) { - throw new Error( + throw new ScriptError( `--sizes only accepts benchmark sizes: ${CODE_GRAPH_WORKSET_FIXTURE_SUPPORTED_SIZES.join(', ')}. Received ${size}.`, ); } @@ -165,22 +166,23 @@ function parseBenchmarkSizes(value: string): readonly CodeGraphWorksetFixtureSiz function integer(value: string | undefined, option: string, minimum: number, maximum: number): number { const parsed = Number(required(value, option)); if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new Error(`${option} must be an integer from ${minimum} through ${maximum}.`); + throw new ScriptError(`${option} must be an integer from ${minimum} through ${maximum}.`); } return parsed; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } function benchmarkCreatedAt(environment: NodeJS.ProcessEnv): string { const epoch = environment.SOURCE_DATE_EPOCH; if (epoch === undefined) return new Date().toISOString(); - if (!/^\d+$/.test(epoch)) throw new Error('SOURCE_DATE_EPOCH must be a non-negative integer number of seconds.'); + if (!/^\d+$/.test(epoch)) + throw new ScriptError('SOURCE_DATE_EPOCH must be a non-negative integer number of seconds.'); const date = new Date(Number(epoch) * 1_000); - if (!Number.isFinite(date.getTime())) throw new Error('SOURCE_DATE_EPOCH is outside the supported date range.'); + if (!Number.isFinite(date.getTime())) throw new ScriptError('SOURCE_DATE_EPOCH is outside the supported date range.'); return date.toISOString(); } @@ -190,4 +192,4 @@ const sourceGit = Effect.fn('benchmarkCodeGraphWorkset.git')((args: readonly str ), ); -if (import.meta.main) BunRuntime.runMain(benchmarkCodeGraphWorkset.pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(benchmarkCodeGraphWorkset, ApplicationLayer)); diff --git a/scripts/benchmark-code-graph.ts b/scripts/benchmark-code-graph.ts index 0e9655cb..0ef53a3a 100644 --- a/scripts/benchmark-code-graph.ts +++ b/scripts/benchmark-code-graph.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Database} from 'bun:sqlite'; import {Clock, Deferred, Effect, Exit, FileSystem, Option, Path, PlatformError} from 'effect'; @@ -216,19 +217,19 @@ export function validateSqliteWriterSettingsEvidence( const phaseEvidence = evidence.filter(settings => settings.benchmarkPhase === benchmarkPhase); const connection = phaseEvidence.filter(settings => settings.phase === 'connection').at(-1); if (!connection || connection.journalMode.toLowerCase() !== 'wal') { - throw new Error(`SQLite writer profile ${profile} did not report a WAL connection for ${benchmarkPhase}.`); + throw new ScriptError(`SQLite writer profile ${profile} did not report a WAL connection for ${benchmarkPhase}.`); } if (requested.mainCacheKiB !== undefined && connection.cacheSizePragma !== -requested.mainCacheKiB) { - throw new Error(`SQLite writer profile ${profile} did not apply its cache size for ${benchmarkPhase}.`); + throw new ScriptError(`SQLite writer profile ${profile} did not apply its cache size for ${benchmarkPhase}.`); } if (requested.mmapSizeBytes !== undefined && connection.mmapSizeBytes !== requested.mmapSizeBytes) { - throw new Error(`SQLite writer profile ${profile} did not apply its mmap size for ${benchmarkPhase}.`); + throw new ScriptError(`SQLite writer profile ${profile} did not apply its mmap size for ${benchmarkPhase}.`); } if ( requested.walAutoCheckpointPages !== undefined && connection.walAutoCheckpointPages !== requested.walAutoCheckpointPages ) { - throw new Error( + throw new ScriptError( `SQLite writer profile ${profile} did not apply its WAL checkpoint cadence for ${benchmarkPhase}.`, ); } @@ -241,7 +242,7 @@ export function validateSqliteWriterSettingsEvidence( (settings, index) => index > building && settings.phase === 'publication' && settings.synchronous === 2, ); if (building < 0 || publication < 0) { - throw new Error( + throw new ScriptError( `SQLite writer profile ${profile} did not restore FULL after NORMAL before ${benchmarkPhase} publication.`, ); } @@ -452,11 +453,13 @@ const benchmarkCodeGraph = Effect.scoped( if (externalPrepared && releaseEvidenceSource) { assertPerformanceControlSet(externalPrepared.externalControls ?? []); if (!externalPrepared.publicRepository) { - return yield* Effect.fail(new Error('Release-bound external evidence requires a public GitHub repository.')); + return yield* Effect.fail( + new ScriptError('Release-bound external evidence requires a public GitHub repository.'), + ); } if (!isReviewedPublicBenchmarkRepository(externalPrepared.publicRepository)) { return yield* Effect.fail( - new Error('Release-bound external evidence requires a reviewed public benchmark repository.'), + new ScriptError('Release-bound external evidence requires a reviewed public benchmark repository.'), ); } } @@ -472,7 +475,7 @@ const benchmarkCodeGraph = Effect.scoped( : undefined; if (options.preflight) { if (!externalPreflight || !externalPrepared) { - return yield* Effect.fail(new Error('External benchmark preflight was not prepared.')); + return yield* Effect.fail(new ScriptError('External benchmark preflight was not prepared.')); } yield* revalidateExternalBenchmarkPreflightState( threadnoteSourceRoot, @@ -491,7 +494,7 @@ const benchmarkCodeGraph = Effect.scoped( } if (externalPrepared && options.retainHomes) { yield* externalPrepared.preserveHomes ?? - Effect.fail(new Error('External benchmark homes could not be retained after preflight.')); + Effect.fail(new ScriptError('External benchmark homes could not be retained after preflight.')); } const runCheckpoint = largeEvidenceRun && options.outputPath @@ -625,7 +628,7 @@ const benchmarkCodeGraph = Effect.scoped( yield* runCheckpoint?.mark('hot-query-and-mutation') ?? Effect.void; if (options.vectors) { if (cold.diagnostics.some(diagnostic => diagnostic.includes('Vector graph retrieval unavailable'))) { - return yield* Effect.fail(new Error(cold.diagnostics.join('\n'))); + return yield* Effect.fail(new ScriptError(cold.diagnostics.join('\n'))); } const semanticControl = yield* query.inspect({ cwd: prepared.repository, @@ -646,7 +649,9 @@ const benchmarkCodeGraph = Effect.scoped( .map(node => `${node.path}:${node.name}:${node.score.toFixed(3)}`) .join(', '); return yield* Effect.fail( - new Error(`Vector benchmark semantic positive control did not resolve; observed ${observed || 'no nodes'}.`), + new ScriptError( + `Vector benchmark semantic positive control did not resolve; observed ${observed || 'no nodes'}.`, + ), ); } } @@ -764,7 +769,7 @@ const benchmarkCodeGraph = Effect.scoped( options.vectors && incremental.diagnostics.some(diagnostic => diagnostic.includes('Vector graph retrieval unavailable')) ) { - return yield* Effect.fail(new Error(incremental.diagnostics.join('\n'))); + return yield* Effect.fail(new ScriptError(incremental.diagnostics.join('\n'))); } if (options.vectors) { const semanticControl = yield* query.inspect({ @@ -790,7 +795,7 @@ const benchmarkCodeGraph = Effect.scoped( .map(node => `${node.path}:${node.name}:${node.score.toFixed(3)}`) .join(', '); return yield* Effect.fail( - new Error( + new ScriptError( `Incremental vector benchmark semantic positive control did not resolve on the new snapshot; ` + `observed ${observed || 'no nodes'}.`, ), @@ -948,7 +953,9 @@ const benchmarkCodeGraph = Effect.scoped( const sameOverlayReferenceTimeline = sameOverlayReference.measurement.timeline; const sameOverlayReferenceTelemetry = sameOverlayReference.telemetry; if (sameOverlayReference.summary.materialization?.mode !== 'full') { - return yield* Effect.fail(new Error('Same-overlay reference build did not execute a full materialization.')); + return yield* Effect.fail( + new ScriptError('Same-overlay reference build did not execute a full materialization.'), + ); } if (prepared.externalCommit) { yield* verifyExternalRepositoryUnchanged(prepared.repository, prepared.externalCommit); @@ -959,7 +966,9 @@ const benchmarkCodeGraph = Effect.scoped( const coldStatusDuration = Number((yield* Clock.currentTimeNanos) - coldStatusStarted) / NANOSECONDS_PER_MILLISECOND; if (!analysisStatus.readySnapshot) { - return yield* Effect.fail(new Error('Code graph benchmark could not resolve its ready snapshot for analysis.')); + return yield* Effect.fail( + new ScriptError('Code graph benchmark could not resolve its ready snapshot for analysis.'), + ); } const managerPerformance = prepared.externalCommit ? yield* benchmarkManagerPerformance( @@ -991,13 +1000,13 @@ const benchmarkCodeGraph = Effect.scoped( analysisCpuDurations.push(cpuMilliseconds(processStarted, processTelemetry()).total); if (result.coverage.topology.state !== 'not-requested' || result.usage.edgeVisits !== 0) { return yield* Effect.fail( - new Error('Code graph benchmark aggregate analysis unexpectedly executed a detail scan.'), + new ScriptError('Code graph benchmark aggregate analysis unexpectedly executed a detail scan.'), ); } analysisComplete = result.coverage.complete; } if (!analysisComplete) { - return yield* Effect.fail(new Error('Code graph benchmark analysis returned partial coverage.')); + return yield* Effect.fail(new ScriptError('Code graph benchmark analysis returned partial coverage.')); } const sameOverlayReferenceAnalysis = yield* analysis.analyze({ databasePath: sameOverlayReferenceLayout.databasePath, @@ -1010,7 +1019,7 @@ const benchmarkCodeGraph = Effect.scoped( sameOverlayReferenceAnalysis.usage.edgeVisits !== 0 ) { return yield* Effect.fail( - new Error('Code graph benchmark reference analysis unexpectedly required a detail scan.'), + new ScriptError('Code graph benchmark reference analysis unexpectedly required a detail scan.'), ); } @@ -1048,7 +1057,7 @@ const benchmarkCodeGraph = Effect.scoped( const incrementalStructuralGraphDigest = incrementalStructuralGraphEvidence.digest; if (coldStructuralGraphDigest === incrementalStructuralGraphDigest) { return yield* Effect.fail( - new Error('The semantic one-file overlay did not change the structural code graph digest.'), + new ScriptError('The semantic one-file overlay did not change the structural code graph digest.'), ); } const sameOverlayReferenceStructuralGraphEvidence = yield* sqliteStructuralGraphEvidence( @@ -1067,7 +1076,9 @@ const benchmarkCodeGraph = Effect.scoped( `${JSON.stringify(structuralGraphParityEvidence, undefined, 2)}\n`, ); } - return yield* Effect.fail(new Error(codeGraphStructuralParityFailureMessage(structuralGraphParityEvidence))); + return yield* Effect.fail( + new ScriptError(codeGraphStructuralParityFailureMessage(structuralGraphParityEvidence)), + ); } const coldLanguageCounts = sqliteLanguageCounts(analysisStatus.databasePath, cold.snapshot.id); const coldWorkspaceScopeRows = sqliteRowCount( @@ -1104,7 +1115,7 @@ const benchmarkCodeGraph = Effect.scoped( [ threadnoteSourceGit(threadnoteSourceRoot, ['rev-parse', 'HEAD']), threadnoteSourceGit(threadnoteSourceRoot, CONFIG_NEUTRAL_GIT_STATUS_ARGUMENTS), - system.hardwareInfo(), + system.hardwareInfo, ], {concurrency: 3}, ); @@ -1496,7 +1507,7 @@ const benchmarkCodeGraph = Effect.scoped( const finalRuntimeProvenance = yield* validateBenchmarkRuntimeProvenance(threadnoteSourceRoot); if (JSON.stringify(finalRuntimeProvenance) !== JSON.stringify(runtimeProvenance)) { return yield* Effect.fail( - new Error('Threadnote benchmark runtime provenance changed during the measured run.'), + new ScriptError('Threadnote benchmark runtime provenance changed during the measured run.'), ); } } @@ -1558,7 +1569,7 @@ export function decodeBenchmarkSource(source: Uint8Array): string { try { return new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}).decode(source); } catch { - throw new Error('The incremental benchmark source must be valid UTF-8 so it can be restored byte-for-byte.'); + throw new ScriptError('The incremental benchmark source must be valid UTF-8 so it can be restored byte-for-byte.'); } } @@ -1571,7 +1582,7 @@ export const applyBenchmarkOverlay = Effect.fn('benchmarkCodeGraph.applyOverlay' const current = yield* fs.readFile(file); if (!sameBytes(current, expectedContents)) { return yield* Effect.fail( - new Error('The benchmark overlay file changed concurrently; Threadnote left the newer contents untouched.'), + new ScriptError('The benchmark overlay file changed concurrently; Threadnote left the newer contents untouched.'), ); } yield* fs.writeFile(file, benchmarkContents); @@ -1586,7 +1597,7 @@ export const restoreBenchmarkOverlay = Effect.fn('benchmarkCodeGraph.restoreOver const current = yield* fs.readFile(file); if (!sameBytes(current, benchmarkContents)) { return yield* Effect.fail( - new Error('The benchmark overlay file changed concurrently; Threadnote left the newer contents untouched.'), + new ScriptError('The benchmark overlay file changed concurrently; Threadnote left the newer contents untouched.'), ); } yield* fs.writeFile(file, originalContents); @@ -1620,7 +1631,7 @@ export function semanticBenchmarkOverlay(filePath: string, source: string): stri if (/(?:^|\/)(?:build(?:\.bazel)?|workspace(?:\.bazel)?|module\.bazel|[^/]+\.(?:bzl|axl))$/.test(normalized)) { return insertAfterBom(source, 'load("@threadnote_benchmark_overlay//:defs.bzl", "threadnote_benchmark_overlay")'); } - throw new Error('The incremental benchmark path must use a supported source language.'); + throw new ScriptError('The incremental benchmark path must use a supported source language.'); } function sourceNewline(source: string): '\n' | '\r\n' { @@ -2401,7 +2412,7 @@ interface ExternalSamplerHandle { } export function parseCodeGraphBenchmarkRunCheckpoint(value: unknown): CodeGraphBenchmarkRunCheckpoint { - if (typeof value !== 'object' || value === null) throw new Error('Benchmark run checkpoint must be an object.'); + if (typeof value !== 'object' || value === null) throw new ScriptError('Benchmark run checkpoint must be an object.'); const checkpoint = value as Partial; if ( checkpoint.version !== 1 || @@ -2411,7 +2422,7 @@ export function parseCodeGraphBenchmarkRunCheckpoint(value: unknown): CodeGraphB typeof checkpoint.updatedAt !== 'string' || !Number.isFinite(Date.parse(checkpoint.updatedAt)) ) { - throw new Error('Benchmark run checkpoint is invalid.'); + throw new ScriptError('Benchmark run checkpoint is invalid.'); } return checkpoint as CodeGraphBenchmarkRunCheckpoint; } @@ -2526,7 +2537,7 @@ export const startExternalSampler = Effect.fn('benchmarkCodeGraph.startExternalS if (Exit.isFailure(stopSignal)) { yield* terminateExternalSampler(subprocess); return yield* Effect.fail( - new Error('Could not signal the code graph benchmark sampler to stop; it was terminated.'), + new ScriptError('Could not signal the code graph benchmark sampler to stop; it was terminated.'), ); } stopped = true; @@ -2538,7 +2549,7 @@ export const startExternalSampler = Effect.fn('benchmarkCodeGraph.startExternalS subprocessExitWithin(subprocess, EXTERNAL_SAMPLER_TERMINATE_TIMEOUT_MS), ); return yield* Effect.fail( - new Error( + new ScriptError( `Code graph benchmark sampler did not stop within ${EXTERNAL_SAMPLER_STOP_TIMEOUT_MS} ms; ` + `it was terminated${exitCode === undefined ? ' without confirming exit' : ''}.`, ), @@ -2547,7 +2558,9 @@ export const startExternalSampler = Effect.fn('benchmarkCodeGraph.startExternalS if (exitCode !== 0) { const stderr = subprocess.stderr ? yield* Effect.promise(() => new Response(subprocess.stderr).text()) : ''; return yield* Effect.fail( - new Error(`Code graph benchmark sampler exited with ${exitCode}: ${stderr.trim() || 'no diagnostic'}`), + new ScriptError( + `Code graph benchmark sampler exited with ${exitCode}: ${stderr.trim() || 'no diagnostic'}`, + ), ); } return parseCodeGraphBenchmarkSamplerArtifact(JSON.parse(yield* fs.readFileString(outputPath))); @@ -2563,11 +2576,11 @@ const waitForExternalSamplerReady = Effect.fn('benchmarkCodeGraph.waitForExterna const startedAt = yield* Clock.currentTimeMillis; while (!(yield* fs.exists(readyPath))) { if (subprocess.exitCode !== null) { - return yield* Effect.fail(new Error(`Code graph benchmark sampler exited before becoming ready.`)); + return yield* Effect.fail(new ScriptError(`Code graph benchmark sampler exited before becoming ready.`)); } if ((yield* Clock.currentTimeMillis) - startedAt >= EXTERNAL_SAMPLER_READY_TIMEOUT_MS) { return yield* Effect.fail( - new Error(`Code graph benchmark sampler was not ready within ${EXTERNAL_SAMPLER_READY_TIMEOUT_MS} ms.`), + new ScriptError(`Code graph benchmark sampler was not ready within ${EXTERNAL_SAMPLER_READY_TIMEOUT_MS} ms.`), ); } yield* Effect.sleep(10); @@ -2671,7 +2684,8 @@ function sqliteRowCount(databasePath: string, query: string, ...parameters: read try { const row = database.query(query).get(...parameters) as {readonly count?: bigint | number} | null; const count = Number(row?.count ?? 0); - if (!Number.isSafeInteger(count) || count < 0) throw new Error(`Invalid SQLite row count for ${databasePath}.`); + if (!Number.isSafeInteger(count) || count < 0) + throw new ScriptError(`Invalid SQLite row count for ${databasePath}.`); return count; } finally { database.close(false); @@ -2691,7 +2705,7 @@ function sqliteLexicalTermRowCount(databasePath: string, snapshotId: string): nu } | null); const count = Number(row?.count ?? 0); if (!Number.isSafeInteger(count) || count < 0) { - throw new Error(`Invalid SQLite lexical term row count for ${databasePath}.`); + throw new ScriptError(`Invalid SQLite lexical term row count for ${databasePath}.`); } return count; } finally { @@ -2733,7 +2747,7 @@ function sqliteGroupedLanguageCounts(rows: readonly unknown[]): ReadonlyMap new Error('Could not open the code graph structural digest read snapshot.', {cause}), + catch: cause => new ScriptError('Could not open the code graph structural digest read snapshot.', {cause}), try: () => openCodeGraphStructuralDigestReadSnapshot(databasePath, snapshotId), }), readSnapshot => @@ -2882,8 +2896,7 @@ export const sqliteStructuralGraphEvidence = Effect.fn('benchmarkCodeGraph.struc }); return yield* readCodeGraphStructuralGraphEvidence(readSnapshot, snapshotId, renewLeaseIfDue); }), - readSnapshot => - Effect.sync(() => closeCodeGraphStructuralDigestReadSnapshot(readSnapshot)).pipe(Effect.catch(() => Effect.void)), + readSnapshot => Effect.sync(() => closeCodeGraphStructuralDigestReadSnapshot(readSnapshot)), ).pipe(Effect.ensuring(store.releaseSnapshotLease(databasePath, lease).pipe(Effect.catch(() => Effect.void)))); }); @@ -2899,7 +2912,8 @@ function openCodeGraphStructuralDigestReadSnapshot( .query('SELECT base_snapshot_id FROM snapshots WHERE id = ? AND state = ? LIMIT 1') .get(snapshotId, 'ready') as {readonly base_snapshot_id?: unknown} | undefined, ); - if (Option.isNone(snapshot)) throw new Error('Ready snapshot was unavailable for the structural graph digest.'); + if (Option.isNone(snapshot)) + throw new ScriptError('Ready snapshot was unavailable for the structural graph digest.'); return { baseSnapshotId: typeof snapshot.value.base_snapshot_id === 'string' @@ -3090,7 +3104,7 @@ const readCodeGraphStructuralGraphEvidence = Effect.fn('benchmarkCodeGraph.readS streamDigest.update('\n'); rowCount += 1; if (!Number.isSafeInteger(rowCount)) { - return yield* Effect.fail(new Error(`Structural digest stream ${stream.name} is too large.`)); + return yield* Effect.fail(new ScriptError(`Structural digest stream ${stream.name} is too large.`)); } } yield* renewLeaseIfDue; @@ -3109,11 +3123,11 @@ export function codeGraphStructuralParityEvidence( referenceStreams.size !== sameOverlayReference.streams.length || incremental.streams.length !== sameOverlayReference.streams.length ) { - throw new Error('Structural graph digest evidence returned an inconsistent stream set.'); + throw new ScriptError('Structural graph digest evidence returned an inconsistent stream set.'); } const mismatchedStreams = incremental.streams.flatMap(stream => { const reference = referenceStreams.get(stream.name); - if (!reference) throw new Error('Structural graph digest evidence returned an inconsistent stream set.'); + if (!reference) throw new ScriptError('Structural graph digest evidence returned an inconsistent stream set.'); return stream.rowCount === reference.rowCount && stream.digest === reference.digest ? [] : [{incremental: stream, name: stream.name, sameOverlayReference: reference}]; @@ -3265,7 +3279,7 @@ const benchmarkExternalQueryControl = Effect.fn('benchmarkCodeGraph.externalQuer duration: EXTERNAL_QUERY_CONTROL_TIMEOUT_MS, orElse: () => Effect.fail( - new Error( + new ScriptError( `External ${phase} query control timed out after ${EXTERNAL_QUERY_CONTROL_TIMEOUT_MS} milliseconds.`, ), ), @@ -3311,7 +3325,9 @@ const benchmarkMcpOperationMatrix = Effect.fn('benchmarkCodeGraph.mcpOperationMa const structuredBytes = encodedBytes(JSON.stringify(response.structuredContent)); const textBytes = encodedBytes(response.text); if (structuredBytes > 24 * 1_024 || textBytes > 24 * 1_024) { - return yield* Effect.fail(new Error(`MCP ${options.operation} output exceeded its 24 KiB per-part budget.`)); + return yield* Effect.fail( + new ScriptError(`MCP ${options.operation} output exceeded its 24 KiB per-part budget.`), + ); } const finished = yield* Clock.currentTimeNanos; results.push({ @@ -3329,7 +3345,7 @@ const benchmarkMcpOperationMatrix = Effect.fn('benchmarkCodeGraph.mcpOperationMa const lexical = yield* execute({operation: 'query', query: queryText}); const seed = lexical.nodes[0]; - if (!seed) return yield* Effect.fail(new Error('MCP operation matrix query returned no seed node.')); + if (!seed) return yield* Effect.fail(new ScriptError('MCP operation matrix query returned no seed node.')); yield* execute({nodeId: seed.id, operation: 'node'}); const neighbors = yield* execute({depth: 1, nodeId: seed.id, operation: 'neighbors'}); yield* execute({operation: 'explain', symbol: seed.id}); @@ -3386,7 +3402,7 @@ export function assertManagerVisualizationBounds( graph.paging.nodeLimit !== limits.nodeLimit || graph.paging.edgeLimit !== limits.edgeLimit ) { - throw new Error(`Manager benchmark ${label} exceeded or misreported its requested graph budget.`); + throw new ScriptError(`Manager benchmark ${label} exceeded or misreported its requested graph budget.`); } } @@ -3406,7 +3422,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage view => view.snapshot.id === expectedSnapshotId && view.snapshot.state === 'ready', ); if (!indexedView) { - return yield* Effect.fail(new Error('Manager benchmark catalog did not expose the expected ready snapshot.')); + return yield* Effect.fail(new ScriptError('Manager benchmark catalog did not expose the expected ready snapshot.')); } const expectedSnapshot = Option.some(expectedSnapshotId); const catalogWarmSamples = Math.max(1, Math.min(samples, 5)); @@ -3443,7 +3459,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage nodeLimit: MANAGER_GRAPH_MAX_NODE_LIMIT, }); if (overviewCold.value.nodes.length === 0) { - return yield* Effect.fail(new Error('Manager benchmark overview returned no graph nodes.')); + return yield* Effect.fail(new ScriptError('Manager benchmark overview returned no graph nodes.')); } for (const sample of overviewWarm) { assertManagerVisualizationBounds('overview warm response', sample.value, { @@ -3452,7 +3468,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage }); } const project = indexedView.projects.find(candidate => (candidate.symbolCount ?? 1) > 0) ?? indexedView.projects[0]; - if (!project) return yield* Effect.fail(new Error('Manager benchmark snapshot has no project detail scope.')); + if (!project) return yield* Effect.fail(new ScriptError('Manager benchmark snapshot has no project detail scope.')); const detailCold = yield* timedJsonEffect( managerGraphVisualization( threadnoteHome, @@ -3467,7 +3483,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage nodeLimit: MANAGER_GRAPH_MAX_NODE_LIMIT, }); if (detailCold.value.nodes.length === 0) { - return yield* Effect.fail(new Error('Manager benchmark selected project detail returned no graph nodes.')); + return yield* Effect.fail(new ScriptError('Manager benchmark selected project detail returned no graph nodes.')); } for (let index = 0; index < warmups; index += 1) { @@ -3495,7 +3511,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage ); const queryResult = querySamples[0]?.value; if (!queryResult || queryResult.nodes.length === 0) { - return yield* Effect.fail(new Error('Manager benchmark bounded query returned no graph nodes.')); + return yield* Effect.fail(new ScriptError('Manager benchmark bounded query returned no graph nodes.')); } for (const sample of querySamples) { assertManagerVisualizationBounds('bounded query response', sample.value, { @@ -3518,7 +3534,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage ); if (rendered.nodes !== renderGraph.nodes.length || rendered.matchedEdges > renderGraph.edges.length) { return yield* Effect.fail( - new Error('Manager benchmark layout-preparation proxy did not preserve its bounded graph input.'), + new ScriptError('Manager benchmark layout-preparation proxy did not preserve its bounded graph input.'), ); } } @@ -3537,7 +3553,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage nodeDetail.value.snapshotId === expectedSnapshotId && staleSnapshotRejected; if (!snapshotBindingPassed) { - return yield* Effect.fail(new Error('Manager benchmark did not preserve exact snapshot binding.')); + return yield* Effect.fail(new ScriptError('Manager benchmark did not preserve exact snapshot binding.')); } const scope = `${indexedView.id}:${expectedSnapshotId}:${queryText}:${MANAGER_QUERY_NODE_LIMIT}:${MANAGER_QUERY_EDGE_LIMIT}`; @@ -3563,7 +3579,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage signal.removeEventListener('abort', cancelOnInterrupt), ); }, - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }); const cancellationGate = createGraphQueryRequestGate(); @@ -3611,7 +3627,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage cancelledOutcome.state === 'cancelled' && acceptedAfterCancellationOutcome.state === 'accepted'; if (!requestCancellationPassed) { - return yield* Effect.fail(new Error('Manager benchmark request-cancellation control failed.')); + return yield* Effect.fail(new ScriptError('Manager benchmark request-cancellation control failed.')); } const lateQueryCompleted = yield* Deferred.make(); @@ -3653,7 +3669,7 @@ const benchmarkManagerPerformanceMeasured = Effect.fn('benchmarkCodeGraph.manage lateOutcome.state === 'stale' && acceptedAfterLateResponseOutcome.state === 'accepted'; if (!staleResponseRejectionPassed) { - return yield* Effect.fail(new Error('Manager benchmark stale-response rejection control failed.')); + return yield* Effect.fail(new ScriptError('Manager benchmark stale-response rejection control failed.')); } return { @@ -3711,7 +3727,7 @@ export const benchmarkManagerPerformance = Effect.fn('benchmarkCodeGraph.manager duration: MANAGER_SEQUENCE_TIMEOUT_MS, orElse: () => Effect.fail( - new Error(`Manager benchmark sequence timed out after ${MANAGER_SEQUENCE_TIMEOUT_MS} milliseconds.`), + new ScriptError(`Manager benchmark sequence timed out after ${MANAGER_SEQUENCE_TIMEOUT_MS} milliseconds.`), ), }), ); @@ -3833,7 +3849,7 @@ export function retainedExternalControlEvidence( const entries = controls .map(control => { const result = resultByLanguage.get(control.expectedLanguage); - if (!result) throw new Error('External control evidence is missing a cold query result.'); + if (!result) throw new ScriptError('External control evidence is missing a cold query result.'); return [ performanceControlMetadataKey(control.expectedLanguage), { @@ -3845,7 +3861,7 @@ export function retainedExternalControlEvidence( }) .sort(([left], [right]) => left.localeCompare(right, 'en')); if (new Set(entries.map(([language]) => language)).size !== entries.length) { - throw new Error('External control evidence contains duplicate public language categories.'); + throw new ScriptError('External control evidence contains duplicate public language categories.'); } return JSON.stringify(Object.fromEntries(entries)); } @@ -3858,7 +3874,7 @@ export function assertPerformanceControlSet(controls: readonly ExternalRepositor actual.length !== expected.length || actual.some((language, index) => language !== expected[index]) ) { - throw new Error( + throw new ScriptError( `Release-bound external performance evidence requires exactly ${PERFORMANCE_CONTROL_LANGUAGES.join(', ')} controls.`, ); } @@ -3882,7 +3898,7 @@ function assertExternalQueryPositiveControl( node => node.path === expected.expectedPath && node.language === expected.expectedLanguage, ); if (result.snapshot.id !== expected.expectedSnapshotId || result.nodes.length === 0 || expectedNodes.length === 0) { - throw new Error( + throw new ScriptError( `External repository ${expected.phase} query did not resolve its expected tracked path and language; ` + 'the query and path were omitted from this diagnostic.', ); @@ -3901,7 +3917,7 @@ function assertPrimaryQueryPositiveControl( phase: 'cold' | 'incremental' | 'same-overlay-reference', ): {readonly digest: string; readonly returnedNodes: number} { if (result.snapshot.id !== expectedSnapshotId || result.nodes.length === 0) { - throw new Error(`Code graph ${phase} primary query returned no current-snapshot nodes.`); + throw new ScriptError(`Code graph ${phase} primary query returned no current-snapshot nodes.`); } return {digest: queryResultStructuralDigest(result), returnedNodes: result.nodes.length}; } @@ -3924,7 +3940,7 @@ export function assertProductionReleaseEvidence(artifact: BenchmarkArtifactV1): function assertProductionLargeEvidence(artifact: BenchmarkArtifactV1, requireReleaseSource = false): void { if (!artifact.suite.startsWith('code-graph-production-large-')) { - throw new Error(`Production release evidence has the wrong suite: ${artifact.suite}.`); + throw new ScriptError(`Production release evidence has the wrong suite: ${artifact.suite}.`); } const measurements = new Map(artifact.measurements.map(measurement => [measurement.name, measurement])); const missing = PRODUCTION_RELEASE_EVIDENCE_MEASUREMENTS.flatMap(required => { @@ -3953,7 +3969,7 @@ function assertProductionLargeEvidence(artifact: BenchmarkArtifactV1, requireRel missing.push(...missingSamplerObservations(measurements)); missing.push(...missingActivationObservations(artifact, measurements)); if (missing.length > 0) { - throw new Error(`Production release evidence is incomplete: ${missing.join(', ')}.`); + throw new ScriptError(`Production release evidence is incomplete: ${missing.join(', ')}.`); } } @@ -4201,7 +4217,7 @@ export const validateBenchmarkRuntimeProvenance = Effect.fn('benchmarkCodeGraph. ); if (!EXACT_GIT_COMMIT_PATTERN.test(sourceCommit) || dirty.length > 0) { return yield* Effect.fail( - new Error('Long code-graph benchmarks require a clean Threadnote checkout at an exact Git commit.'), + new ScriptError('Long code-graph benchmarks require a clean Threadnote checkout at an exact Git commit.'), ); } const environment = system.environment(); @@ -4219,7 +4235,7 @@ export const validateBenchmarkRuntimeProvenance = Effect.fn('benchmarkCodeGraph. !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(githubRepository ?? '') ) { return yield* Effect.fail( - new Error('GitHub Actions benchmark provenance is incomplete or does not match the checkout commit.'), + new ScriptError('GitHub Actions benchmark provenance is incomplete or does not match the checkout commit.'), ); } const [realSourceRoot, realGithubWorkspace, sourceLockfileSha256, sourcePackageManifestSha256] = yield* Effect.all( @@ -4234,7 +4250,7 @@ export const validateBenchmarkRuntimeProvenance = Effect.fn('benchmarkCodeGraph. const normalize = (value: string) => system.platform === 'win32' ? path.resolve(value).toLocaleLowerCase('en-US') : path.resolve(value); if (normalize(realSourceRoot) !== normalize(realGithubWorkspace)) { - return yield* Effect.fail(new Error('GitHub Actions benchmark provenance is not bound to this workspace.')); + return yield* Effect.fail(new ScriptError('GitHub Actions benchmark provenance is not bound to this workspace.')); } yield* verifyBenchmarkSourceUnchanged(sourceRoot, sourceCommit); return { @@ -4262,11 +4278,11 @@ export const revalidateExternalBenchmarkPreflightState = Effect.fn( expectedRuntimeProvenance: BenchmarkRuntimeProvenance | undefined, ) { if (!expectedExternalCommit || !expectedRuntimeProvenance) { - return yield* Effect.fail(new Error('External benchmark preflight has incomplete provenance.')); + return yield* Effect.fail(new ScriptError('External benchmark preflight has incomplete provenance.')); } const runtimeProvenance = yield* validateBenchmarkRuntimeProvenance(sourceRoot); if (JSON.stringify(runtimeProvenance) !== JSON.stringify(expectedRuntimeProvenance)) { - return yield* Effect.fail(new Error('Threadnote benchmark runtime provenance changed during preflight.')); + return yield* Effect.fail(new ScriptError('Threadnote benchmark runtime provenance changed during preflight.')); } yield* verifyExternalRepositoryUnchanged(externalRepository, expectedExternalCommit); // Keep the source checkout check last so no artifact is emitted after a @@ -4318,7 +4334,7 @@ export function resolvedReleaseEvidenceSource( checkoutCommit !== sha || dirty ) { - throw new Error( + throw new ScriptError( 'Release benchmark provenance requires a locally resolvable tag, its exact commit SHA, and a clean checkout.', ); } @@ -4338,7 +4354,7 @@ const validateReleaseEvidenceSource = Effect.fn('benchmarkCodeGraph.validateRele !EXACT_GIT_COMMIT_PATTERN.test(sha) ) { return yield* Effect.fail( - new Error('Release benchmark provenance requires a Threadnote 4 release tag and its exact commit SHA.'), + new ScriptError('Release benchmark provenance requires a Threadnote 4 release tag and its exact commit SHA.'), ); } const [commit, dirty, resolvedSha] = yield* Effect.all( @@ -4414,7 +4430,7 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG else if (argument === '--incremental-path') incrementalPath = required(args[++index], argument); else if (argument === '--materialization-transaction-batches') { const value = integer(args[++index], argument, 1); - if (value !== 1 && value !== 4) throw new Error(`${argument} must be 1 or 4.`); + if (value !== 1 && value !== 4) throw new ScriptError(`${argument} must be 1 or 4.`); materializationTransactionBatchLimit = value; } else if (argument === '--minimum-free-gib') minimumFreeGiB = integer(args[++index], argument, 1); else if (argument === '--model-home') modelHome = required(args[++index], argument); @@ -4423,7 +4439,7 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG else if (argument === '--repository') repository = required(args[++index], argument); else if (argument === '--profile') { const value = required(args[++index], argument); - if (value !== 'production-large') throw new Error(`Unknown code graph benchmark profile: ${value}`); + if (value !== 'production-large') throw new ScriptError(`Unknown code graph benchmark profile: ${value}`); profile = value; } else if (argument === '--profile-files') profileFiles = integer(args[++index], argument, 2); else if (argument === '--profile-symbols') profileSymbols = integer(args[++index], argument, 2); @@ -4432,7 +4448,7 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG else if (argument === '--sqlite-writer-profile') { const value = required(args[++index], argument); if (!(value in CODE_GRAPH_SQLITE_WRITER_PROFILES)) { - throw new Error(`Unknown SQLite writer benchmark profile: ${value}`); + throw new ScriptError(`Unknown SQLite writer benchmark profile: ${value}`); } sqliteWriterProfile = value as CodeGraphSqliteWriterProfile; } else if (argument === '--warmups') warmups = integer(args[++index], argument, 0); @@ -4440,35 +4456,37 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG else if (argument === '--preflight') preflight = true; else if (argument === '--retain-homes') retainHomes = true; else if (argument === '--vectors') vectors = true; - else throw new Error(`Unknown code graph benchmark option: ${argument}`); + else throw new ScriptError(`Unknown code graph benchmark option: ${argument}`); } - if (!/^code-graph-[a-z0-9-]+$/.test(fixture)) throw new Error(`Invalid code graph fixture name: ${fixture}.`); + if (!/^code-graph-[a-z0-9-]+$/.test(fixture)) throw new ScriptError(`Invalid code graph fixture name: ${fixture}.`); if (vectors && fixture !== 'code-graph-v1') { - throw new Error('The vector semantic control is currently defined only for code-graph-v1.'); + throw new ScriptError('The vector semantic control is currently defined only for code-graph-v1.'); } if (profile && scaleSymbols !== undefined) { - throw new Error('--profile and --scale-symbols are separate fixture modes and cannot be combined.'); + throw new ScriptError('--profile and --scale-symbols are separate fixture modes and cannot be combined.'); } if ((profileFiles !== undefined || profileSymbols !== undefined) && profile !== 'production-large') { - throw new Error('--profile-files and --profile-symbols require --profile production-large.'); + throw new ScriptError('--profile-files and --profile-symbols require --profile production-large.'); } if (profile === 'production-large' && fixture !== 'code-graph-v1') { - throw new Error('The production-large profile uses the code-graph-v1 query contract.'); + throw new ScriptError('The production-large profile uses the code-graph-v1 query contract.'); } if (profile === 'production-large' && failOnBudget) { - throw new Error( + throw new ScriptError( 'The opt-in production-large profile has no portable latency budget; retain and review its artifact.', ); } if (sqliteWriterProfile !== undefined && sqliteWriterProfile !== 'current' && failOnBudget) { - throw new Error('SQLite writer candidate runs retain comparison evidence and cannot use production budgets.'); + throw new ScriptError('SQLite writer candidate runs retain comparison evidence and cannot use production budgets.'); } const legacyControlValues = [queryText, expectedPath, expectedLanguage].filter(value => value !== undefined).length; if (structuredControls.length > 0 && legacyControlValues > 0) { - throw new Error('--control cannot be combined with legacy --query, --expected-path, or --expected-language flags.'); + throw new ScriptError( + '--control cannot be combined with legacy --query, --expected-path, or --expected-language flags.', + ); } if (legacyControlValues > 0 && legacyControlValues < 3) { - throw new Error( + throw new ScriptError( 'Legacy external control flags require --query, --expected-path, and --expected-language together.', ); } @@ -4479,23 +4497,25 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG ? [{expectedLanguage, expectedPath, query: queryText}] : []; if (new Set(externalControls.map(control => control.expectedLanguage)).size !== externalControls.length) { - throw new Error('External query controls must use unique language categories.'); + throw new ScriptError('External query controls must use unique language categories.'); } if (repository !== undefined) { if (profile !== undefined || scaleSymbols !== undefined || vectors) { - throw new Error('--repository cannot be combined with generated profiles, scale fixtures, or vectors.'); + throw new ScriptError('--repository cannot be combined with generated profiles, scale fixtures, or vectors.'); } if (!incrementalPath || externalControls.length === 0 || !outputPath) { - throw new Error('--repository requires --incremental-path, at least one --control, and --output.'); + throw new ScriptError('--repository requires --incremental-path, at least one --control, and --output.'); } if (failOnBudget) { - throw new Error('External repositories retain same-runner evidence and do not use portable latency budgets.'); + throw new ScriptError( + 'External repositories retain same-runner evidence and do not use portable latency budgets.', + ); } if ((homePath === undefined) !== (referenceHomePath === undefined)) { - throw new Error('--home and --reference-home must be provided together.'); + throw new ScriptError('--home and --reference-home must be provided together.'); } if (retainHomes && (homePath === undefined || referenceHomePath === undefined)) { - throw new Error('--retain-homes requires explicit --home and --reference-home paths.'); + throw new ScriptError('--retain-homes requires explicit --home and --reference-home paths.'); } } else if ( incrementalPath !== undefined || @@ -4505,7 +4525,7 @@ export function parseCodeGraphBenchmarkArguments(args: readonly string[]): CodeG retainHomes || preflight ) { - throw new Error( + throw new ScriptError( '--incremental-path, external controls, benchmark homes, --retain-homes, and --preflight require --repository.', ); } @@ -4542,17 +4562,17 @@ function parseExternalRepositoryQueryControl(value: string): ExternalRepositoryQ try { parsed = JSON.parse(value); } catch { - throw new Error('--control must be a JSON object with query, expectedPath, and expectedLanguage strings.'); + throw new ScriptError('--control must be a JSON object with query, expectedPath, and expectedLanguage strings.'); } if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { - throw new Error('--control must be a JSON object with query, expectedPath, and expectedLanguage strings.'); + throw new ScriptError('--control must be a JSON object with query, expectedPath, and expectedLanguage strings.'); } const candidate = parsed as Partial>; const query = typeof candidate.query === 'string' ? candidate.query.trim() : ''; const expectedPath = typeof candidate.expectedPath === 'string' ? candidate.expectedPath.trim() : ''; const expectedLanguage = typeof candidate.expectedLanguage === 'string' ? candidate.expectedLanguage.trim() : ''; if (!query || !expectedPath || !/^[a-z][a-z0-9-]*$/.test(expectedLanguage)) { - throw new Error( + throw new ScriptError( '--control requires non-empty query and expectedPath strings plus a lowercase expectedLanguage category.', ); } @@ -4565,7 +4585,7 @@ const prepareExternalCodeGraphFixture = Effect.fn('benchmarkCodeGraph.prepareExt const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (!options.repository || !options.incrementalPath || options.externalControls.length === 0 || !options.outputPath) { - return yield* Effect.fail(new Error('External repository benchmark options are incomplete.')); + return yield* Effect.fail(new ScriptError('External repository benchmark options are incomplete.')); } const requestedRoot = path.resolve(options.repository); const repository = yield* fs.realPath( @@ -4580,10 +4600,10 @@ const prepareExternalCodeGraphFixture = Effect.fn('benchmarkCodeGraph.prepareExt {concurrency: 3}, ); if (!EXACT_GIT_COMMIT_PATTERN.test(externalCommit)) { - return yield* Effect.fail(new Error('External repository did not resolve to an exact Git commit.')); + return yield* Effect.fail(new ScriptError('External repository did not resolve to an exact Git commit.')); } if (dirty.length > 0) { - return yield* Effect.fail(new Error('External repository benchmark requires a clean checkout.')); + return yield* Effect.fail(new ScriptError('External repository benchmark requires a clean checkout.')); } const publicRepository = publicGitHubRepositoryEvidence(origin); const publicRepositoryVerification = yield* verifyPublicRepositoryCommit( @@ -4601,7 +4621,9 @@ const prepareExternalCodeGraphFixture = Effect.fn('benchmarkCodeGraph.prepareExt !artifactContainment.startsWith(`..${path.sep}`)) ) { return yield* Effect.fail( - new Error('--output must be outside the external repository so benchmark evidence cannot modify the checkout.'), + new ScriptError( + '--output must be outside the external repository so benchmark evidence cannot modify the checkout.', + ), ); } @@ -4644,7 +4666,7 @@ const prepareExternalCodeGraphFixture = Effect.fn('benchmarkCodeGraph.prepareExt const home = homeReservation.home; const referenceHome = referenceHomeReservation.home; if (home === referenceHome) { - return yield* Effect.fail(new Error('Primary and same-overlay reference benchmark homes must be different.')); + return yield* Effect.fail(new ScriptError('Primary and same-overlay reference benchmark homes must be different.')); } for (const benchmarkHome of [home, referenceHome]) { const containment = path.relative(repository, benchmarkHome); @@ -4652,7 +4674,7 @@ const prepareExternalCodeGraphFixture = Effect.fn('benchmarkCodeGraph.prepareExt containment === '' || (!path.isAbsolute(containment) && containment !== '..' && !containment.startsWith(`..${path.sep}`)) ) { - return yield* Effect.fail(new Error('Benchmark homes must be outside the external repository.')); + return yield* Effect.fail(new ScriptError('Benchmark homes must be outside the external repository.')); } } return { @@ -4683,7 +4705,7 @@ export function publicGitHubRepositoryEvidence(remote: string): PublicGitHubRepo try { parsed = new URL(trimmed); } catch { - throw new Error('External benchmark origin must be a public GitHub repository URL.'); + throw new ScriptError('External benchmark origin must be a public GitHub repository URL.'); } const allowedSshUser = parsed.protocol === 'ssh:' && (parsed.username.length === 0 || parsed.username === 'git'); @@ -4697,10 +4719,10 @@ export function publicGitHubRepositoryEvidence(remote: string): PublicGitHubRepo parsed.hash.length > 0 || !['https:', 'ssh:'].includes(parsed.protocol) ) { - throw new Error('External benchmark origin must be a public GitHub repository URL.'); + throw new ScriptError('External benchmark origin must be a public GitHub repository URL.'); } const match = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+?)(?:\.git)?\/?$/.exec(parsed.pathname); - if (!match) throw new Error('External benchmark origin must be a public GitHub repository URL.'); + if (!match) throw new ScriptError('External benchmark origin must be a public GitHub repository URL.'); return [match[1]!, match[2]!] as const; })(); const name = `${owner}/${repository}`; @@ -4760,16 +4782,16 @@ const exactCommitProofRemote = Effect.fn('benchmarkCodeGraph.exactCommitProofRem environment.THREADNOTE_BENCHMARK_RELEASE_SHA?.trim() ) { return yield* Effect.fail( - new Error('The local public-repository proof seam is test-only and unavailable for release evidence.'), + new ScriptError('The local public-repository proof seam is test-only and unavailable for release evidence.'), ); } if (!path.isAbsolute(testRemote)) { - return yield* Effect.fail(new Error('The local public-repository proof seam requires an absolute Git path.')); + return yield* Effect.fail(new ScriptError('The local public-repository proof seam requires an absolute Git path.')); } const resolved = yield* fs.realPath(testRemote); const info = yield* fs.stat(resolved); if (info.type !== 'Directory') { - return yield* Effect.fail(new Error('The local public-repository proof seam requires a Git directory.')); + return yield* Effect.fail(new ScriptError('The local public-repository proof seam requires a Git directory.')); } return resolved; }); @@ -4782,7 +4804,7 @@ export const verifyAnonymousPublicGitHubRepository = Effect.fn( environment: Readonly> = process.env, ) { if (!EXACT_GIT_COMMIT_PATTERN.test(externalCommit)) { - return yield* Effect.fail(new Error('External repository proof requires an exact Git commit.')); + return yield* Effect.fail(new ScriptError('External repository proof requires an exact Git commit.')); } const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -4829,7 +4851,7 @@ export const verifyAnonymousPublicGitHubRepository = Effect.fn( ), Effect.mapError( () => - new Error( + new ScriptError( 'External benchmark commit could not be fetched from the public repository through credentials-disabled anonymous HTTPS.', ), ), @@ -4841,10 +4863,14 @@ export const verifyAnonymousPublicGitHubRepository = Effect.fn( 'FETCH_HEAD^{commit}', ]).pipe( Effect.map(result => result.stdout.trim()), - Effect.mapError(() => new Error('External benchmark public-repository proof did not resolve the fetched commit.')), + Effect.mapError( + () => new ScriptError('External benchmark public-repository proof did not resolve the fetched commit.'), + ), ); if (resolved !== externalCommit) { - return yield* Effect.fail(new Error('External benchmark public-repository proof resolved a different commit.')); + return yield* Effect.fail( + new ScriptError('External benchmark public-repository proof resolved a different commit.'), + ); } return 'anonymous-https-exact-commit-fetch' as const; }); @@ -4874,7 +4900,7 @@ const acquireFreshBenchmarkHome = Effect.fn('benchmarkCodeGraph.acquireFreshHome containment === '' || (!path.isAbsolute(containment) && containment !== '..' && !containment.startsWith(`..${path.sep}`)) ) { - return yield* Effect.fail(new Error('Benchmark homes must be outside the external repository.')); + return yield* Effect.fail(new ScriptError('Benchmark homes must be outside the external repository.')); } const parent = path.dirname(target); yield* fs.makeDirectory(parent, {mode: 0o700, recursive: true}); @@ -4882,7 +4908,7 @@ const acquireFreshBenchmarkHome = Effect.fn('benchmarkCodeGraph.acquireFreshHome const exclusiveTarget = path.join(canonicalParent, path.basename(target)); return yield* Effect.acquireRelease( fs.makeDirectory(exclusiveTarget, {mode: 0o700}).pipe( - Effect.mapError(() => new Error('Explicit benchmark home paths must be fresh and exclusively reservable.')), + Effect.mapError(() => new ScriptError('Explicit benchmark home paths must be fresh and exclusively reservable.')), Effect.andThen( fs.realPath(exclusiveTarget).pipe( Effect.flatMap(home => { @@ -4891,7 +4917,7 @@ const acquireFreshBenchmarkHome = Effect.fn('benchmarkCodeGraph.acquireFreshHome (!path.isAbsolute(finalContainment) && finalContainment !== '..' && !finalContainment.startsWith(`..${path.sep}`)) - ? Effect.fail(new Error('Benchmark homes must be outside the external repository.')) + ? Effect.fail(new ScriptError('Benchmark homes must be outside the external repository.')) : Effect.succeed(home); }), ), @@ -4926,14 +4952,14 @@ const externalBenchmarkPreflight = Effect.fn('benchmarkCodeGraph.externalPreflig const system = yield* SystemInfo; if (!externalBenchmarkPlatformSupported(process.platform)) { return yield* Effect.fail( - new Error('External code-graph evidence currently requires Linux or macOS process and storage telemetry.'), + new ScriptError('External code-graph evidence currently requires Linux or macOS process and storage telemetry.'), ); } if (!prepared.externalCommit || !prepared.incrementalSourcePath || !prepared.referenceHome) { - return yield* Effect.fail(new Error('External benchmark preflight requires a complete prepared fixture.')); + return yield* Effect.fail(new ScriptError('External benchmark preflight requires a complete prepared fixture.')); } if (!runtimeProvenance) { - return yield* Effect.fail(new Error('External benchmark preflight requires exact runtime provenance.')); + return yield* Effect.fail(new ScriptError('External benchmark preflight requires exact runtime provenance.')); } const source = decodeBenchmarkSource( yield* fs.readFile(path.join(prepared.repository, prepared.incrementalSourcePath)), @@ -4944,14 +4970,14 @@ const externalBenchmarkPreflight = Effect.fn('benchmarkCodeGraph.externalPreflig repositoryGit(prepared.repository, ['rev-parse', 'HEAD^{tree}']).pipe(Effect.map(result => result.stdout.trim())), filesystemCapacity(prepared.home), filesystemCapacity(prepared.referenceHome), - system.hardwareInfo(), + system.hardwareInfo, ], {concurrency: 4}, ); const minimumFreeBytes = minimumFreeGiB * 1_073_741_824; if (primaryCapacity.availableBytes < minimumFreeBytes || referenceCapacity.availableBytes < minimumFreeBytes) { return yield* Effect.fail( - new Error( + new ScriptError( `External benchmark preflight requires at least ${minimumFreeGiB} GiB free on every benchmark-home filesystem.`, ), ); @@ -4990,11 +5016,13 @@ const verifyPublicRepositoryOrigin = Effect.fn('benchmarkCodeGraph.verifyPublicR const remote = (yield* repositoryGit(repository, ['remote', 'get-url', 'origin'])).stdout.trim(); const actual = publicGitHubRepositoryEvidence(remote); if (actual.name !== expected.name || actual.url !== expected.url) { - return yield* Effect.fail(new Error('External benchmark public repository identity changed during the run.')); + return yield* Effect.fail(new ScriptError('External benchmark public repository identity changed during the run.')); } const verification = yield* verifyPublicRepositoryCommit(actual, externalCommit, process.env); if (verification !== expectedVerification) { - return yield* Effect.fail(new Error('External benchmark public repository verification changed during the run.')); + return yield* Effect.fail( + new ScriptError('External benchmark public repository verification changed during the run.'), + ); } }); @@ -5006,7 +5034,7 @@ const filesystemCapacity = Effect.fn('benchmarkCodeGraph.filesystemCapacity')(fu const availableKilobytes = Number(columns[capacityIndex - 1] ?? Number.NaN); const filesystem = columns[0] ?? ''; if (!filesystem || capacityIndex < 3 || !Number.isSafeInteger(availableKilobytes) || availableKilobytes < 0) { - return yield* Effect.fail(new Error('Could not determine benchmark filesystem capacity.')); + return yield* Effect.fail(new ScriptError('Could not determine benchmark filesystem capacity.')); } return {availableBytes: availableKilobytes * 1_024, filesystem}; }); @@ -5080,7 +5108,9 @@ export const benchmarkConcurrentWorktreeIsolation = Effect.fn('benchmarkCodeGrap } for (const target of [...repositoryRoots, root]) { if (yield* fs.exists(target)) { - return yield* Effect.fail(new Error('Concurrent worktree benchmark cleanup left a generated path behind.')); + return yield* Effect.fail( + new ScriptError('Concurrent worktree benchmark cleanup left a generated path behind.'), + ); } } }); @@ -5124,7 +5154,7 @@ export const benchmarkConcurrentWorktreeIsolation = Effect.fn('benchmarkCodeGrap {concurrency: 2}, ); if (options.failureInjection === 'after-index') { - return yield* Effect.fail(new Error('Injected concurrent worktree benchmark failure after indexing.')); + return yield* Effect.fail(new ScriptError('Injected concurrent worktree benchmark failure after indexing.')); } const [primaryQuery, linkedQuery, primaryCrossQuery, linkedCrossQuery] = yield* Effect.all( [ @@ -5180,7 +5210,7 @@ export const benchmarkConcurrentWorktreeIsolation = Effect.fn('benchmarkCodeGrap !primaryCrossQuery.nodes.some(node => node.name === 'linkedWorktreeSentinel') && !linkedCrossQuery.nodes.some(node => node.name === 'primaryWorktreeSentinel'); if (!isolationPassed) { - return yield* Effect.fail(new Error('Concurrent linked-worktree graph isolation control failed.')); + return yield* Effect.fail(new ScriptError('Concurrent linked-worktree graph isolation control failed.')); } const durationMilliseconds = Math.max( Number.EPSILON, @@ -5200,7 +5230,9 @@ export const benchmarkConcurrentWorktreeIsolation = Effect.fn('benchmarkCodeGrap duration: WORKTREE_ISOLATION_TIMEOUT_MS, orElse: () => Effect.fail( - new Error(`Concurrent worktree control timed out after ${WORKTREE_ISOLATION_TIMEOUT_MS} milliseconds.`), + new ScriptError( + `Concurrent worktree control timed out after ${WORKTREE_ISOLATION_TIMEOUT_MS} milliseconds.`, + ), ), }), Effect.ensuring(cleanup.pipe(Effect.orDie)), @@ -5291,7 +5323,7 @@ const validateExternalTrackedRegularPath = Effect.fn('benchmarkCodeGraph.validat option: '--control expectedPath' | '--incremental-path', ) { if (path.isAbsolute(value)) { - return yield* Effect.fail(new Error(`${option} must name a repository-relative file.`)); + return yield* Effect.fail(new ScriptError(`${option} must name a repository-relative file.`)); } const normalized = path.normalize(value); const source = path.resolve(repository, normalized); @@ -5302,7 +5334,7 @@ const validateExternalTrackedRegularPath = Effect.fn('benchmarkCodeGraph.validat containment.startsWith(`..${path.sep}`) || path.isAbsolute(containment) ) { - return yield* Effect.fail(new Error(`${option} must name a repository-relative file.`)); + return yield* Effect.fail(new ScriptError(`${option} must name a repository-relative file.`)); } const canonicalSource = yield* fs.realPath(source); const canonicalContainment = path.relative(repository, canonicalSource); @@ -5312,16 +5344,18 @@ const validateExternalTrackedRegularPath = Effect.fn('benchmarkCodeGraph.validat canonicalContainment.startsWith(`..${path.sep}`) || path.isAbsolute(canonicalContainment) ) { - return yield* Effect.fail(new Error(`${option} resolved outside the external repository.`)); + return yield* Effect.fail(new ScriptError(`${option} resolved outside the external repository.`)); } const gitPath = containment.split(path.sep).join('/'); const tracked = yield* repositoryGit(repository, ['ls-files', '--stage', '--error-unmatch', '--', gitPath]); if (!/^100(?:644|755)\s/.test(tracked.stdout)) { - return yield* Effect.fail(new Error(`${option} must name a tracked regular file, not a link or submodule.`)); + return yield* Effect.fail( + new ScriptError(`${option} must name a tracked regular file, not a link or submodule.`), + ); } const info = yield* fs.stat(source); if (info.type !== 'File') { - return yield* Effect.fail(new Error(`${option} must name a tracked regular file.`)); + return yield* Effect.fail(new ScriptError(`${option} must name a tracked regular file.`)); } return gitPath; }, @@ -5340,7 +5374,7 @@ const verifyExternalRepositoryUnchanged = Effect.fn('benchmarkCodeGraph.verifyEx ); if (commit !== expectedCommit || dirty.length > 0) { return yield* Effect.fail( - new Error( + new ScriptError( 'External repository changed during the benchmark; its evidence was rejected after restoring the overlay.', ), ); @@ -5360,7 +5394,7 @@ const verifyBenchmarkSourceUnchanged = Effect.fn('benchmarkCodeGraph.verifyBench ); if (commit !== expectedCommit || dirty.length > 0) { return yield* Effect.fail( - new Error('Threadnote source changed during the external benchmark; its evidence was not published.'), + new ScriptError('Threadnote source changed during the external benchmark; its evidence was not published.'), ); } }); @@ -5384,7 +5418,7 @@ const canonicalizeProspectivePath = Effect.fn('benchmarkCodeGraph.canonicalizePr if (Option.isSome(canonical)) return path.join(canonical.value, ...suffix); const parent = path.dirname(current); if (parent === current) { - return yield* Effect.fail(new Error(`Could not resolve an existing parent for output path ${target}.`)); + return yield* Effect.fail(new ScriptError(`Could not resolve an existing parent for output path ${target}.`)); } suffix.unshift(path.basename(current)); current = parent; @@ -5392,7 +5426,7 @@ const canonicalizeProspectivePath = Effect.fn('benchmarkCodeGraph.canonicalizePr }); export function productionProfile(options: CodeGraphBenchmarkOptions): ProductionCodeGraphFixtureProfile { - if (options.profile !== 'production-large') throw new Error('Production fixture profile was not selected.'); + if (options.profile !== 'production-large') throw new ScriptError('Production fixture profile was not selected.'); if (options.profileFiles === undefined && options.profileSymbols === undefined) { return PRODUCTION_LARGE_CODE_GRAPH_PROFILE; } @@ -5436,7 +5470,7 @@ export function productionProfile(options: CodeGraphBenchmarkOptions): Productio const metadataGraphSymbols = workspaceCount + 3; const declarationSymbols = targetGraphSymbols - sourceFiles - metadataGraphSymbols; if (declarationSymbols < sourceFiles) { - throw new Error( + throw new ScriptError( '--profile-symbols must cover the requested files, manifest/module symbols, and at least one declaration per file.', ); } @@ -5559,7 +5593,7 @@ export function enforceCodeGraphBenchmarkBudget( value: unknown, scaleSymbols: number | undefined, ): void { - if (typeof value !== 'object' || value === null) throw new Error('Code graph budget file must be an object.'); + if (typeof value !== 'object' || value === null) throw new ScriptError('Code graph budget file must be an object.'); const record = value as { readonly developmentPerformance?: unknown; readonly developmentPerformanceByPlatform?: Readonly>; @@ -5589,7 +5623,7 @@ export function enforceCodeGraphBenchmarkBudget( ? {...baseSelected, ...platformOverride} : baseSelected; if (typeof selected !== 'object' || selected === null) { - throw new Error( + throw new ScriptError( `No reviewed ${artifact.metadata.vectorEnabled === true ? 'vector ' : ''}code graph performance budget exists ` + `for ${scaleSymbols ?? 'development'}.`, ); @@ -5624,7 +5658,7 @@ export function enforceCodeGraphBenchmarkBudget( failures.push(`${measurementName} ${statistic} exceeds ${maximum}`); } } - if (failures.length > 0) throw new Error(`Code graph performance budget failed: ${failures.join('; ')}`); + if (failures.length > 0) throw new ScriptError(`Code graph performance budget failed: ${failures.join('; ')}`); } function processPeakRssBytes(): number { @@ -5657,13 +5691,13 @@ const prepareBenchmarkEmbedding = Effect.fn('benchmarkCodeGraph.prepareEmbedding function integer(value: string | undefined, option: string, minimum: number): number { const parsed = Number.parseInt(required(value, option), 10); - if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`${option} must be at least ${minimum}`); + if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new ScriptError(`${option} must be at least ${minimum}`); return parsed; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } -if (import.meta.main) BunRuntime.runMain(benchmarkCodeGraph.pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(benchmarkCodeGraph, ApplicationLayer)); diff --git a/scripts/benchmark-recall-micro.ts b/scripts/benchmark-recall-micro.ts index 0ab7e87b..60480cb5 100644 --- a/scripts/benchmark-recall-micro.ts +++ b/scripts/benchmark-recall-micro.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {bench, do_not_optimize, run} from 'mitata'; import {Effect} from 'effect'; @@ -29,7 +30,7 @@ const benchmarkRecall = Effect.gen(function* () { const emitJson = arguments_.includes('--json') || outputIndex !== -1; const result = yield* Effect.tryPromise({ try: () => run({format: emitJson ? 'quiet' : undefined, throw: true}), - catch: cause => new Error('Recall microbenchmark failed.', {cause}), + catch: cause => new ScriptError('Recall microbenchmark failed.', {cause}), }); if (!emitJson) return; @@ -40,7 +41,7 @@ const benchmarkRecall = Effect.gen(function* () { Effect.forEach([...fixtures], ([size, fixture]) => fixtureHash(JSON.stringify(fixture)).pipe(Effect.map(hash => [String(size), hash] as const)), ), - system.hardwareInfo(), + system.hardwareInfo, ], {concurrency: 'unbounded'}, ); @@ -80,7 +81,7 @@ const benchmarkRecall = Effect.gen(function* () { }; if (outputIndex !== -1) { const outputPath = arguments_[outputIndex + 1]; - if (!outputPath) return yield* Effect.fail(new Error('--output requires a path')); + if (!outputPath) return yield* Effect.fail(new ScriptError('--output requires a path')); yield* atomicWrite(outputPath, `${JSON.stringify(artifact, undefined, 2)}\n`); } if (arguments_.includes('--json') || outputIndex === -1) { @@ -92,4 +93,4 @@ const git = Effect.fn('benchmark.git')((arguments_: readonly string[]) => runCommandEffect('git', arguments_, {timeoutMs: 30_000}).pipe(Effect.map(result => result.stdout.trim())), ); -BunRuntime.runMain(benchmarkRecall.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(benchmarkRecall, ApplicationLayer)); diff --git a/scripts/benchmark-recall-v2.ts b/scripts/benchmark-recall-v2.ts index ac6eadc1..98a982d5 100644 --- a/scripts/benchmark-recall-v2.ts +++ b/scripts/benchmark-recall-v2.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Clock, Effect} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; @@ -46,7 +47,7 @@ const benchmarkRecall = Effect.gen(function* () { const result = runQuery(); const finishedAt = yield* Clock.currentTimeNanos; durations.push(Number(finishedAt - startedAt) / NANOSECONDS_PER_MILLISECOND); - if (!result.results[0]) return yield* Effect.fail(new Error('Recall benchmark returned no result')); + if (!result.results[0]) return yield* Effect.fail(new ScriptError('Recall benchmark returned no result')); const memory = system.memoryUsage(); rss.push(memory.rss); externalMemory.push(memory.external); @@ -56,7 +57,7 @@ const benchmarkRecall = Effect.gen(function* () { const latency = benchmarkMeasurement('hybrid-rank-one-query', 'milliseconds', durations); const throughput = durations.map(duration => 1_000 / duration); const [commit, status, hardware] = yield* Effect.all( - [git(['rev-parse', 'HEAD']), git(['status', '--porcelain']), system.hardwareInfo()], + [git(['rev-parse', 'HEAD']), git(['status', '--porcelain']), system.hardwareInfo], { concurrency: 'unbounded', }, @@ -124,7 +125,7 @@ function parseArguments(args: readonly string[]): BenchmarkOptions { else if (argument === '--samples') samples = positiveInteger(args[++index], argument); else if (argument === '--seed') seed = positiveInteger(args[++index], argument); else if (argument === '--warmups') warmups = nonNegativeInteger(args[++index], argument); - else throw new Error(`Unknown recall benchmark option: ${argument}`); + else throw new ScriptError(`Unknown recall benchmark option: ${argument}`); } return {documentCount, outputPath, samples, seed, warmups}; } @@ -135,21 +136,21 @@ const git = Effect.fn('benchmark.git')((arguments_: readonly string[]) => function positiveInteger(value: string | undefined, option: string): number { const parsed = nonNegativeInteger(value, option); - if (parsed < 1) throw new Error(`${option} requires a positive integer`); + if (parsed < 1) throw new ScriptError(`${option} requires a positive integer`); return parsed; } function nonNegativeInteger(value: string | undefined, option: string): number { const parsed = Number.parseInt(requiredValue(value, option), 10); if (!Number.isSafeInteger(parsed) || parsed < 0) { - throw new Error(`${option} requires a non-negative integer`); + throw new ScriptError(`${option} requires a non-negative integer`); } return parsed; } function requiredValue(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } -BunRuntime.runMain(benchmarkRecall.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(benchmarkRecall, ApplicationLayer)); diff --git a/scripts/benchmark-recall-vectors.ts b/scripts/benchmark-recall-vectors.ts index 8e898f18..74115afe 100644 --- a/scripts/benchmark-recall-vectors.ts +++ b/scripts/benchmark-recall-vectors.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunServices from '@effect/platform-bun/BunServices'; import {BunRuntime} from '@effect/platform-bun'; import {Database} from 'bun:sqlite'; @@ -28,7 +29,7 @@ const options = parseOptions(process.argv.slice(2)); const modelStoreLayer = Layer.succeed( LocalModelStore, LocalModelStore.of({ - install: () => Effect.die(new Error('Unexpected model installation')), + install: () => Effect.die(new ScriptError('Unexpected model installation')), path: home => `${home}/models/benchmark.gguf`, remove: () => Effect.succeed(false), status: home => Effect.succeed(modelInstallation(home)), @@ -39,11 +40,11 @@ const modelStoreLayer = Layer.succeed( const runtimeLayer = Layer.succeed( LocalModelRuntime, LocalModelRuntime.of({ - diagnostics: () => Effect.succeed({backend: 'fake', buildType: 'prebuilt', cpuMathCores: 4}), + diagnostics: Effect.succeed({backend: 'fake', buildType: 'prebuilt', cpuMathCores: 4}), embedMany: ({inputs, manifest: requested}) => Effect.sync(() => inputs.map(input => deterministicVector(requested.dimensions ?? 0, input))), - generate: () => Effect.die(new Error('Unexpected generation')), - rerank: () => Effect.die(new Error('Unexpected reranking')), + generate: () => Effect.die(new ScriptError('Unexpected generation')), + rerank: () => Effect.die(new ScriptError('Unexpected reranking')), }), ); @@ -103,7 +104,7 @@ const program = Effect.scoped( const expectedUri = candidates[targetIndex]!.uri; if (scores?.size !== expectedSize || (scores.get(expectedUri) ?? -1) < 0.999) { return yield* Effect.fail( - new Error( + new ScriptError( `Vector benchmark returned ${scores?.size ?? 0}/${expectedSize} results without the exact target ${expectedUri}.`, ), ); @@ -209,40 +210,44 @@ const program = Effect.scoped( const scale = options.documents / DEFAULT_DOCUMENT_COUNT; const boundedScale = Math.max(1, scale); if (result.scenarios.semanticQuery.p95Milliseconds > boundedScale * 750) { - return yield* Effect.fail(new Error('Semantic vector query exceeded its linear scale budget.')); + return yield* Effect.fail(new ScriptError('Semantic vector query exceeded its linear scale budget.')); } if (result.scenarios.initialBuild.milliseconds > boundedScale * 15_000) { - return yield* Effect.fail(new Error('Initial vector build exceeded its linear scale budget.')); + return yield* Effect.fail(new ScriptError('Initial vector build exceeded its linear scale budget.')); } if (result.scenarios.incrementalBuild.milliseconds > boundedScale * 3_000) { - return yield* Effect.fail(new Error('Incremental vector build exceeded its linear scale budget.')); + return yield* Effect.fail(new ScriptError('Incremental vector build exceeded its linear scale budget.')); } if (result.scenarios.initialBuild.peakRssBytes > boundedScale * 768 * MEBIBYTE) { - return yield* Effect.fail(new Error('Initial vector build exceeded its bounded-memory budget.')); + return yield* Effect.fail(new ScriptError('Initial vector build exceeded its bounded-memory budget.')); } if (result.scenarios.incrementalBuild.peakRssBytes > boundedScale * 768 * MEBIBYTE) { - return yield* Effect.fail(new Error('Incremental vector build exceeded its bounded-memory budget.')); + return yield* Effect.fail(new ScriptError('Incremental vector build exceeded its bounded-memory budget.')); } if (result.scenarios.semanticQuery.rssDeltaBytes > 128 * MEBIBYTE) { - return yield* Effect.fail(new Error('Semantic vector query exceeded its bounded-memory budget.')); + return yield* Effect.fail(new ScriptError('Semantic vector query exceeded its bounded-memory budget.')); } if (!storageBudget.databaseBytesWithinBudget) { - return yield* Effect.fail(new Error('Vector database exceeded its per-document storage budget.')); + return yield* Effect.fail(new ScriptError('Vector database exceeded its per-document storage budget.')); } if (!storageBudget.incrementalCompactedBytesWithinBudget) { - return yield* Effect.fail(new Error('Incremental vector build caused unexpected compacted database growth.')); + return yield* Effect.fail( + new ScriptError('Incremental vector build caused unexpected compacted database growth.'), + ); } if (incremental.embeddedChunkCount !== 1 || incremental.reusedChunkCount !== options.documents - 1) { - return yield* Effect.fail(new Error('Incremental vector build did not reuse all unchanged chunks.')); + return yield* Effect.fail(new ScriptError('Incremental vector build did not reuse all unchanged chunks.')); } if (storage.vectorValues !== options.documents || storage.chunkMappings !== options.documents) { - return yield* Effect.fail(new Error('Content-addressed vector storage duplicated unchanged vector values.')); + return yield* Effect.fail( + new ScriptError('Content-addressed vector storage duplicated unchanged vector values.'), + ); } } }), ); -BunRuntime.runMain(program.pipe(Effect.provide(benchmarkLayer))); +BunRuntime.runMain(provideScriptLayer(program, benchmarkLayer)); function deterministicVector(dimensions: number, input: string): readonly number[] { const numeric = Number(/(\d+)(?!.*\d)/.exec(input)?.[1] ?? 0); @@ -300,13 +305,13 @@ function parseOptions(arguments_: readonly string[]): { else if (argument === '--samples') samples = positiveInteger(arguments_[++index], '--samples'); else if (argument === '--output') output = arguments_[++index]; else if (argument === '--fail-on-budget') failOnBudget = true; - else throw new Error(`Unknown vector benchmark option: ${argument}`); + else throw new ScriptError(`Unknown vector benchmark option: ${argument}`); } return {...(output ? {output} : {}), documents, failOnBudget, samples}; } function positiveInteger(value: string | undefined, option: string): number { const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new Error(`${option} requires a positive integer.`); + if (!Number.isSafeInteger(parsed) || parsed <= 0) throw new ScriptError(`${option} requires a positive integer.`); return parsed; } diff --git a/scripts/benchmark-worktree-readiness.ts b/scripts/benchmark-worktree-readiness.ts index ede85a9e..42196f49 100644 --- a/scripts/benchmark-worktree-readiness.ts +++ b/scripts/benchmark-worktree-readiness.ts @@ -1,8 +1,10 @@ -import {execFileSync, spawnSync} from 'node:child_process'; -import {createHash} from 'node:crypto'; -import {appendFileSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync} from 'node:fs'; -import {cpus, platform, release, tmpdir, totalmem} from 'node:os'; -import {basename, dirname, join, resolve} from 'node:path'; +import {provideScriptLayer, ScriptError} from './effect/errors.js'; +import * as BunRuntime from '@effect/platform-bun/BunRuntime'; +import * as BunServices from '@effect/platform-bun/BunServices'; +import {Clock, Console, Effect, FileSystem, Layer, Path} from 'effect'; +import {CommandExecutor, runCommandEffect} from '../src/effect/command.js'; +import {sha256HexSync} from '../src/crypto/sha256.js'; +import {runtimeHostHardwareInfo, runtimeOperatingSystemRelease, SystemInfo} from '../src/effect/system.js'; const DEFAULT_CANDIDATE_REF = 'v4.0.1'; const DEFAULT_SAMPLES = 5; @@ -77,147 +79,176 @@ interface ScenarioEvidence { readonly queryParityPassed: true; } -const options = parseArguments(process.argv.slice(2)); -const repositoryRoot = gitTopLevel(process.cwd()); -const candidateCommit = git(repositoryRoot, ['rev-parse', '--verify', `${options.candidateRef}^{commit}`]); -const baselineRef = options.baselineRef ?? `${candidateCommit}^`; -const baselineCommit = git(repositoryRoot, ['rev-parse', '--verify', `${baselineRef}^{commit}`]); -if (candidateCommit === baselineCommit) throw new Error('Candidate and baseline commits must differ.'); -git(repositoryRoot, ['merge-base', '--is-ancestor', baselineCommit, candidateCommit]); - -const temporaryRoot = mkdtempSync(join(tmpdir(), 'threadnote-worktree-readiness-')); -const runtimeRoots: string[] = []; -try { - progress(`Preparing exact runtime checkouts in ${temporaryRoot}`); - const baseline = prepareRuntime('baseline', baselineCommit); - const candidate = prepareRuntime('candidate', candidateCommit); - const fixtures = { - baseline: prepareFixture(baseline), - candidate: prepareFixture(candidate), - } as const; - - progress('Building the shared warm anchor for each runtime'); - const anchorObservations = { - baseline: runIndex(baseline, fixtures.baseline.primary), - candidate: runIndex(candidate, fixtures.candidate.primary), - } as const; - assertParity(anchorObservations.baseline, anchorObservations.candidate, 'warm anchor'); - - const scenarios = { - graphEquivalentCommit: runScenario('graphEquivalentCommit', fixtures), - oneFileChange: runScenario('oneFileChange', fixtures), - } satisfies Record; - - const artifact = { - schemaVersion: 1, - generatedAt: new Date().toISOString(), - scope: { - name: 'warm-linked-worktree-lexical-readiness', - description: - 'Elapsed wall time from invoking graph index in a newly linked worktree until a current lexical snapshot is ready.', - excludes: ['cold anchor construction', 'dependency installation', 'optional vector enrichment'], - order: 'Alternating same-machine runs; candidate first for even samples and baseline first for odd samples.', - samples: options.samples, - warmups: options.warmups, - }, - source: { - repository: 'Kashkovsky/threadnote', - repositoryUrl: 'https://github.com/Kashkovsky/threadnote', - fixtureCommit: candidateCommit, - candidate: {commit: candidateCommit, ref: options.candidateRef}, - baseline: {commit: baselineCommit, ref: baselineRef}, - harness: { - path: 'scripts/benchmark-worktree-readiness.ts', - sha256: sha256(readFileSync(new URL(import.meta.url))), +interface BenchmarkContext { + readonly baselineCommit: string; + readonly candidateCommit: string; + readonly options: BenchmarkOptions; + readonly repositoryRoot: string; + readonly temporaryRoot: string; +} + +const benchmarkWorktreeReadiness = Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const system = yield* SystemInfo; + const options = parseArguments(system.processArguments.slice(2)); + const repositoryRoot = yield* gitTopLevel(system.currentDirectory()); + const candidateCommit = yield* git(repositoryRoot, ['rev-parse', '--verify', `${options.candidateRef}^{commit}`]); + const baselineRef = options.baselineRef ?? `${candidateCommit}^`; + const baselineCommit = yield* git(repositoryRoot, ['rev-parse', '--verify', `${baselineRef}^{commit}`]); + if (candidateCommit === baselineCommit) { + return yield* Effect.fail(new ScriptError('Candidate and baseline commits must differ.')); + } + yield* git(repositoryRoot, ['merge-base', '--is-ancestor', baselineCommit, candidateCommit]); + + const temporaryRoot = yield* fs.makeTempDirectoryScoped({prefix: 'threadnote-worktree-readiness-'}); + const context: BenchmarkContext = {baselineCommit, candidateCommit, options, repositoryRoot, temporaryRoot}; + yield* progress(`Preparing exact runtime checkouts in ${temporaryRoot}`); + const baseline = yield* prepareRuntime(context, 'baseline', baselineCommit); + const candidate = yield* prepareRuntime(context, 'candidate', candidateCommit); + const fixtures = { + baseline: yield* prepareFixture(context, baseline), + candidate: yield* prepareFixture(context, candidate), + } as const; + + yield* progress('Building the shared warm anchor for each runtime'); + const anchorObservations = { + baseline: yield* runIndex(baseline, fixtures.baseline.primary), + candidate: yield* runIndex(candidate, fixtures.candidate.primary), + } as const; + assertParity(anchorObservations.baseline, anchorObservations.candidate, 'warm anchor'); + + const scenarios = { + graphEquivalentCommit: yield* runScenario(context, 'graphEquivalentCommit', fixtures), + oneFileChange: yield* runScenario(context, 'oneFileChange', fixtures), + } satisfies Record; + + const hardware = runtimeHostHardwareInfo(); + const harnessPath = yield* path.fromFileUrl(new URL(import.meta.url)); + + const artifact = { + schemaVersion: 1, + generatedAt: new Date(yield* Clock.currentTimeMillis).toISOString(), + scope: { + name: 'warm-linked-worktree-lexical-readiness', + description: + 'Elapsed wall time from invoking graph index in a newly linked worktree until a current lexical snapshot is ready.', + excludes: ['cold anchor construction', 'dependency installation', 'optional vector enrichment'], + order: 'Alternating same-machine runs; candidate first for even samples and baseline first for odd samples.', + samples: options.samples, + warmups: options.warmups, }, - }, - environment: { - architecture: process.arch, - bun: Bun.version, - cpu: cpus()[0]?.model ?? 'unknown', - logicalCpuCount: cpus().length, - memoryBytes: totalmem(), - operatingSystem: `${platform()} ${release()}`, - parserWorkers: PARSER_WORKERS, - runner: 'same-machine-local-source', - }, - anchor: { - graphParityPassed: true, - baseline: anchorObservations.baseline.graph, - candidate: anchorObservations.candidate.graph, - }, - scenarios, - } as const; - - validateArtifact(artifact); - const json = `${JSON.stringify(artifact, undefined, 2)}\n`; - if (options.outputPath) atomicWrite(resolve(options.outputPath), json); - process.stdout.write(json); -} finally { - for (const runtimeRoot of runtimeRoots.reverse()) { - rmSync(runtimeRoot, {force: true, recursive: true}); - } - rmSync(temporaryRoot, {force: true, recursive: true}); -} + source: { + repository: 'Kashkovsky/threadnote', + repositoryUrl: 'https://github.com/Kashkovsky/threadnote', + fixtureCommit: candidateCommit, + candidate: {commit: candidateCommit, ref: options.candidateRef}, + baseline: {commit: baselineCommit, ref: baselineRef}, + harness: { + path: 'scripts/benchmark-worktree-readiness.ts', + sha256: sha256HexSync(yield* fs.readFile(harnessPath)), + }, + }, + environment: { + architecture: system.architecture, + bun: system.runtimeVersion, + cpu: hardware.cpuModel, + logicalCpuCount: hardware.logicalCpuCount, + memoryBytes: hardware.memoryBytes, + operatingSystem: `${system.platform} ${runtimeOperatingSystemRelease}`, + parserWorkers: PARSER_WORKERS, + runner: 'same-machine-local-source', + }, + anchor: { + graphParityPassed: true, + baseline: anchorObservations.baseline.graph, + candidate: anchorObservations.candidate.graph, + }, + scenarios, + } as const; -function prepareRuntime(name: RuntimeName, commit: string): RuntimeCheckout { - const root = join(temporaryRoot, `runtime-${name}`); - runtimeRoots.push(root); - cloneAtCommit(repositoryRoot, root, commit); - progress(`Installing frozen dependencies for ${name} ${commit.slice(0, 12)}`); - command('bun', ['install', '--frozen-lockfile', '--ignore-scripts'], root, 5 * 60 * 1_000); - if (git(root, ['status', '--porcelain', '--untracked-files=no']) !== '') { - throw new Error(`${name} runtime checkout changed during dependency installation.`); + validateArtifact(context, artifact); + const json = `${JSON.stringify(artifact, undefined, 2)}\n`; + if (options.outputPath) yield* atomicWrite(path.resolve(options.outputPath), json); + yield* Console.log(JSON.stringify(artifact, undefined, 2)); + }), +); + +const prepareRuntime = Effect.fn('worktreeReadiness.prepareRuntime')(function* ( + context: BenchmarkContext, + name: RuntimeName, + commit: string, +) { + const path = yield* Path.Path; + const root = path.join(context.temporaryRoot, `runtime-${name}`); + yield* cloneAtCommit(context, context.repositoryRoot, root, commit); + yield* progress(`Installing frozen dependencies for ${name} ${commit.slice(0, 12)}`); + yield* command('bun', ['install', '--frozen-lockfile', '--ignore-scripts'], root, 5 * 60 * 1_000); + if ((yield* git(root, ['status', '--porcelain', '--untracked-files=no'])) !== '') { + return yield* Effect.fail(new ScriptError(`${name} runtime checkout changed during dependency installation.`)); } return { commit, - home: join(temporaryRoot, `home-${name}`), + home: path.join(context.temporaryRoot, `home-${name}`), name, root, }; -} - -function prepareFixture(runtime: RuntimeCheckout): FixtureCheckout { - const primary = join(temporaryRoot, `fixture-${runtime.name}`); - cloneAtCommit(repositoryRoot, primary, candidateCommit); - git(primary, ['switch', '--create', 'worktree-readiness-benchmark']); - git(primary, ['config', 'user.name', 'Threadnote Benchmark']); - git(primary, ['config', 'user.email', 'benchmark@threadnote.local']); +}); + +const prepareFixture = Effect.fn('worktreeReadiness.prepareFixture')(function* ( + context: BenchmarkContext, + runtime: RuntimeCheckout, +) { + const path = yield* Path.Path; + const primary = path.join(context.temporaryRoot, `fixture-${runtime.name}`); + yield* cloneAtCommit(context, context.repositoryRoot, primary, context.candidateCommit); + yield* git(primary, ['switch', '--create', 'worktree-readiness-benchmark']); + yield* git(primary, ['config', 'user.name', 'Threadnote Benchmark']); + yield* git(primary, ['config', 'user.email', 'benchmark@threadnote.local']); return { primary, runtime, - worktreeRoot: join(temporaryRoot, `worktrees-${runtime.name}`), + worktreeRoot: path.join(context.temporaryRoot, `worktrees-${runtime.name}`), }; -} +}); -function runScenario( +const runScenario = Effect.fn('worktreeReadiness.runScenario')(function* ( + context: BenchmarkContext, scenario: ScenarioName, fixtures: Readonly>, -): ScenarioEvidence { +) { const observations: Record = {baseline: [], candidate: []}; - const totalRuns = options.warmups + options.samples; + const totalRuns = context.options.warmups + context.options.samples; for (let run = 0; run < totalRuns; run += 1) { - const measured = run >= options.warmups; - const sample = run - options.warmups; + const measured = run >= context.options.warmups; + const sample = run - context.options.warmups; const logicalRun = `${scenario}-${run + 1}`; - for (const fixture of Object.values(fixtures)) prepareScenarioCommit(fixture, scenario, run); + for (const fixture of Object.values(fixtures)) yield* prepareScenarioCommit(fixture, scenario, run); const worktrees = { - baseline: addLinkedWorktree(fixtures.baseline, logicalRun), - candidate: addLinkedWorktree(fixtures.candidate, logicalRun), + baseline: yield* addLinkedWorktree(fixtures.baseline, logicalRun), + candidate: yield* addLinkedWorktree(fixtures.candidate, logicalRun), } as const; const order: readonly RuntimeName[] = run % 2 === 0 ? ['candidate', 'baseline'] : ['baseline', 'candidate']; const current: Partial> = {}; - try { + yield* Effect.gen(function* () { for (const name of order) { - progress( - `${scenario} ${measured ? `sample ${sample + 1}/${options.samples}` : `warmup ${run + 1}/${options.warmups}`} · ${name}`, + yield* progress( + `${scenario} ${ + measured + ? `sample ${sample + 1}/${context.options.samples}` + : `warmup ${run + 1}/${context.options.warmups}` + } · ${name}`, ); - current[name] = runIndex(fixtures[name].runtime, worktrees[name]); + current[name] = yield* runIndex(fixtures[name].runtime, worktrees[name]); } - } finally { - removeLinkedWorktree(fixtures.baseline, worktrees.baseline); - removeLinkedWorktree(fixtures.candidate, worktrees.candidate); - } + }).pipe( + Effect.ensuring( + removeLinkedWorktree(fixtures.baseline, worktrees.baseline).pipe( + Effect.andThen(removeLinkedWorktree(fixtures.candidate, worktrees.candidate)), + ), + ), + ); const baseline = requireObservation(current.baseline, `${logicalRun} baseline`); const candidate = requireObservation(current.candidate, `${logicalRun} candidate`); assertParity(baseline, candidate, logicalRun); @@ -227,8 +258,14 @@ function runScenario( observations.candidate.push(candidate); } } - const baseline = summarize(observations.baseline.map(value => value.durationMilliseconds)); - const candidate = summarize(observations.candidate.map(value => value.durationMilliseconds)); + const baseline = summarize( + observations.baseline.map(value => value.durationMilliseconds), + context.options.samples, + ); + const candidate = summarize( + observations.candidate.map(value => value.durationMilliseconds), + context.options.samples, + ); const medianSpeedup = baseline.medianMilliseconds / candidate.medianMilliseconds; return { baseline: { @@ -248,53 +285,69 @@ function runScenario( percentFaster: (1 - candidate.medianMilliseconds / baseline.medianMilliseconds) * 100, queryParityPassed: true, }; -} +}); -function prepareScenarioCommit(fixture: FixtureCheckout, scenario: ScenarioName, run: number): void { +const prepareScenarioCommit = Effect.fn('worktreeReadiness.prepareScenarioCommit')(function* ( + fixture: FixtureCheckout, + scenario: ScenarioName, + run: number, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const system = yield* SystemInfo; if (scenario === 'oneFileChange') { - appendFileSync( - join(fixture.primary, FIXTURE_QUERY_PATH), + yield* fs.writeFileString( + path.join(fixture.primary, FIXTURE_QUERY_PATH), `\n// Threadnote worktree-readiness benchmark sample ${run + 1}.\n`, + {flag: 'a'}, ); - git(fixture.primary, ['add', FIXTURE_QUERY_PATH]); + yield* git(fixture.primary, ['add', FIXTURE_QUERY_PATH]); } const date = new Date(Date.UTC(2026, 7, 4, scenario === 'graphEquivalentCommit' ? 1 : 2, run, 0)).toISOString(); const environment = { - ...process.env, + ...system.environment(), GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date, }; - command( + yield* command( 'git', ['commit', '--allow-empty', '--quiet', '--message', `${scenario} benchmark sample ${run + 1}`], fixture.primary, 30_000, environment, ); -} - -function addLinkedWorktree(fixture: FixtureCheckout, logicalRun: string): string { - const worktree = join(fixture.worktreeRoot, logicalRun); - git(fixture.primary, ['worktree', 'add', '--quiet', '--detach', worktree, 'HEAD']); +}); + +const addLinkedWorktree = Effect.fn('worktreeReadiness.addLinkedWorktree')(function* ( + fixture: FixtureCheckout, + logicalRun: string, +) { + const path = yield* Path.Path; + const worktree = path.join(fixture.worktreeRoot, logicalRun); + yield* git(fixture.primary, ['worktree', 'add', '--quiet', '--detach', worktree, 'HEAD']); return worktree; -} - -function removeLinkedWorktree(fixture: FixtureCheckout, worktree: string): void { - if (!worktree.startsWith(`${fixture.worktreeRoot}/`)) { - throw new Error(`Refusing to remove an unexpected worktree path: ${worktree}`); +}); + +const removeLinkedWorktree = Effect.fn('worktreeReadiness.removeLinkedWorktree')(function* ( + fixture: FixtureCheckout, + worktree: string, +) { + const path = yield* Path.Path; + if (!worktree.startsWith(`${fixture.worktreeRoot}${path.sep}`)) { + return yield* Effect.fail(new ScriptError(`Refusing to remove an unexpected worktree path: ${worktree}`)); } - git(fixture.primary, ['worktree', 'remove', '--force', worktree]); -} + yield* git(fixture.primary, ['worktree', 'remove', '--force', worktree]); +}); -function runIndex(runtime: RuntimeCheckout, cwd: string): Observation { - const started = process.hrtime.bigint(); - const output = runThreadnote(runtime, ['--log-level', 'none', 'graph', 'index', '--cwd', cwd, '--json']); - const durationMilliseconds = Number(process.hrtime.bigint() - started) / 1_000_000; +const runIndex = Effect.fn('worktreeReadiness.runIndex')(function* (runtime: RuntimeCheckout, cwd: string) { + const started = yield* Clock.currentTimeNanos; + const output = yield* runThreadnote(runtime, ['--log-level', 'none', 'graph', 'index', '--cwd', cwd, '--json']); + const durationMilliseconds = Number((yield* Clock.currentTimeNanos) - started) / 1_000_000; const summary = finalJsonRecord(output, 'code-graph-index'); const snapshot = record(summary.snapshot, 'snapshot'); const materialization = record(summary.materialization, 'materialization'); const query = finalJsonRecord( - runThreadnote(runtime, [ + yield* runThreadnote(runtime, [ '--log-level', 'none', 'graph', @@ -312,7 +365,7 @@ function runIndex(runtime: RuntimeCheckout, cwd: string): Observation { ); const nodes = array(query.nodes, 'query.nodes').map(value => record(value, 'query node')); if (!nodes.some(node => node.name === FIXTURE_QUERY && node.path === FIXTURE_QUERY_PATH)) { - throw new Error(`${runtime.name} query control did not return ${FIXTURE_QUERY_PATH}#${FIXTURE_QUERY}.`); + throw new ScriptError(`${runtime.name} query control did not return ${FIXTURE_QUERY_PATH}#${FIXTURE_QUERY}.`); } return { durationMilliseconds, @@ -327,52 +380,55 @@ function runIndex(runtime: RuntimeCheckout, cwd: string): Observation { stagedFiles: integerField(materialization, 'stagedFiles'), totalFiles: integerField(materialization, 'totalFiles'), }; -} +}); -function runThreadnote(runtime: RuntimeCheckout, arguments_: readonly string[]): string { - const result = spawnSync(process.execPath, [join(runtime.root, 'src/standalone.ts'), ...arguments_], { - cwd: runtime.root, - encoding: 'utf8', - env: { - ...process.env, - FORCE_COLOR: '0', - NO_COLOR: '1', - THREADNOTE_CODE_GRAPH_PARSER_WORKERS: String(PARSER_WORKERS), - THREADNOTE_HOME: runtime.home, +const runThreadnote = Effect.fn('worktreeReadiness.runThreadnote')(function* ( + runtime: RuntimeCheckout, + arguments_: readonly string[], +) { + const path = yield* Path.Path; + const system = yield* SystemInfo; + const result = yield* runCommandEffect( + system.executablePath, + [path.join(runtime.root, 'src/standalone.ts'), ...arguments_], + { + cwd: runtime.root, + env: { + ...system.environment(), + FORCE_COLOR: '0', + NO_COLOR: '1', + THREADNOTE_CODE_GRAPH_PARSER_WORKERS: String(PARSER_WORKERS), + THREADNOTE_HOME: runtime.home, + }, + maxOutputBytes: MAX_COMMAND_OUTPUT_BYTES, + timeoutMs: INDEX_TIMEOUT_MILLISECONDS, }, - maxBuffer: MAX_COMMAND_OUTPUT_BYTES, - timeout: INDEX_TIMEOUT_MILLISECONDS, - }); - if (result.status !== 0 || result.error) { - throw new Error( - `${runtime.name} Threadnote command failed: ${(result.error?.message ?? result.stderr.trim()) || `exit ${result.status}`}`, - ); - } + ); return result.stdout; -} +}); function assertExpectedMode(scenario: ScenarioName, baseline: Observation, candidate: Observation): void { if (baseline.materializationMode !== 'full') { - throw new Error(`${scenario} baseline unexpectedly used ${baseline.materializationMode}.`); + throw new ScriptError(`${scenario} baseline unexpectedly used ${baseline.materializationMode}.`); } const expectedCandidate = scenario === 'graphEquivalentCommit' ? 'reused-snapshot' : 'incremental-clean'; if (candidate.materializationMode !== expectedCandidate) { - throw new Error(`${scenario} candidate unexpectedly used ${candidate.materializationMode}.`); + throw new ScriptError(`${scenario} candidate unexpectedly used ${candidate.materializationMode}.`); } if (scenario === 'graphEquivalentCommit' && candidate.stagedFiles !== 0) { - throw new Error('Graph-equivalent candidate commit staged files instead of aliasing the ready graph.'); + throw new ScriptError('Graph-equivalent candidate commit staged files instead of aliasing the ready graph.'); } if (scenario === 'oneFileChange' && candidate.stagedFiles !== 1) { - throw new Error(`One-file candidate commit staged ${candidate.stagedFiles} files instead of one.`); + throw new ScriptError(`One-file candidate commit staged ${candidate.stagedFiles} files instead of one.`); } } function assertParity(baseline: Observation, candidate: Observation, label: string): void { if (JSON.stringify(baseline.graph) !== JSON.stringify(candidate.graph)) { - throw new Error(`${label} graph counts differ between the baseline and candidate.`); + throw new ScriptError(`${label} graph counts differ between the baseline and candidate.`); } if (baseline.queryDigest !== candidate.queryDigest) { - throw new Error(`${label} query control differs between the baseline and candidate.`); + throw new ScriptError(`${label} query control differs between the baseline and candidate.`); } } @@ -399,16 +455,16 @@ function queryEvidenceDigest(query: Record): string { targetName: edge.targetName, })) .sort(compareJson); - return sha256(JSON.stringify({edges, nodes})); + return sha256HexSync(JSON.stringify({edges, nodes})); } function compareJson(left: unknown, right: unknown): number { return JSON.stringify(left).localeCompare(JSON.stringify(right), 'en'); } -function summarize(values: readonly number[]): Summary { - if (values.length !== options.samples || values.some(value => !Number.isFinite(value) || value <= 0)) { - throw new Error(`Expected ${options.samples} positive benchmark observations.`); +function summarize(values: readonly number[], expectedSamples: number): Summary { + if (values.length !== expectedSamples || values.some(value => !Number.isFinite(value) || value <= 0)) { + throw new ScriptError(`Expected ${expectedSamples} positive benchmark observations.`); } const sorted = [...values].sort((left, right) => left - right); const middle = Math.floor(sorted.length / 2); @@ -421,20 +477,27 @@ function summarize(values: readonly number[]): Summary { }; } -function validateArtifact(artifact: { - readonly scenarios: Readonly>; - readonly source: {readonly baseline: {readonly commit: string}; readonly candidate: {readonly commit: string}}; -}): void { - if (artifact.source.baseline.commit !== baselineCommit || artifact.source.candidate.commit !== candidateCommit) { - throw new Error('Benchmark artifact source provenance drifted during the run.'); +function validateArtifact( + context: BenchmarkContext, + artifact: { + readonly scenarios: Readonly>; + readonly source: {readonly baseline: {readonly commit: string}; readonly candidate: {readonly commit: string}}; + }, +): void { + if ( + artifact.source.baseline.commit !== context.baselineCommit || + artifact.source.candidate.commit !== context.candidateCommit + ) { + throw new ScriptError('Benchmark artifact source provenance drifted during the run.'); } for (const [name, scenario] of Object.entries(artifact.scenarios)) { - if (!scenario.graphParityPassed || !scenario.queryParityPassed) throw new Error(`${name} parity did not pass.`); + if (!scenario.graphParityPassed || !scenario.queryParityPassed) + throw new ScriptError(`${name} parity did not pass.`); if (!Number.isFinite(scenario.medianSpeedup) || scenario.medianSpeedup <= 1) { - throw new Error(`${name} did not improve median readiness time.`); + throw new ScriptError(`${name} did not improve median readiness time.`); } if (!Number.isFinite(scenario.percentFaster) || scenario.percentFaster <= 0 || scenario.percentFaster >= 100) { - throw new Error(`${name} has an invalid percentage improvement.`); + throw new ScriptError(`${name} has an invalid percentage improvement.`); } } } @@ -446,45 +509,60 @@ function finalJsonRecord(output: string, expectedType?: string): Record JSON.parse(line) as unknown) .filter(value => value !== null && typeof value === 'object' && !Array.isArray(value)) as Record[]; const selected = expectedType ? records.findLast(record => record.type === expectedType) : records.at(-1); - if (!selected) throw new Error(`Threadnote command did not emit ${expectedType ?? 'a final JSON record'}.`); + if (!selected) throw new ScriptError(`Threadnote command did not emit ${expectedType ?? 'a final JSON record'}.`); return selected; } -function cloneAtCommit(source: string, target: string, commit: string): void { - command('git', ['clone', '--quiet', '--no-local', '--no-checkout', source, target], temporaryRoot, 2 * 60 * 1_000); - git(target, ['checkout', '--quiet', '--detach', commit]); - if (git(target, ['rev-parse', 'HEAD']) !== commit) throw new Error(`Could not prepare exact checkout ${commit}.`); -} +const cloneAtCommit = Effect.fn('worktreeReadiness.cloneAtCommit')(function* ( + context: BenchmarkContext, + source: string, + target: string, + commit: string, +) { + yield* command( + 'git', + ['clone', '--quiet', '--no-local', '--no-checkout', source, target], + context.temporaryRoot, + 2 * 60 * 1_000, + ); + yield* git(target, ['checkout', '--quiet', '--detach', commit]); + if ((yield* git(target, ['rev-parse', 'HEAD'])) !== commit) { + return yield* Effect.fail(new ScriptError(`Could not prepare exact checkout ${commit}.`)); + } +}); -function git(cwd: string, arguments_: readonly string[]): string { - return command('git', arguments_, cwd, 2 * 60 * 1_000).trim(); -} +const git = Effect.fn('worktreeReadiness.git')(function* (cwd: string, arguments_: readonly string[]) { + return (yield* command('git', arguments_, cwd, 2 * 60 * 1_000)).trim(); +}); -function gitTopLevel(cwd: string): string { - return git(cwd, ['rev-parse', '--show-toplevel']); -} +const gitTopLevel = Effect.fn('worktreeReadiness.gitTopLevel')(function* (cwd: string) { + return yield* git(cwd, ['rev-parse', '--show-toplevel']); +}); -function command( +const command = Effect.fn('worktreeReadiness.command')(function* ( executable: string, arguments_: readonly string[], cwd: string, timeout: number, - env: NodeJS.ProcessEnv = process.env, -): string { - return execFileSync(executable, arguments_, { + env?: NodeJS.ProcessEnv, +) { + const result = yield* runCommandEffect(executable, arguments_, { cwd, - encoding: 'utf8', env, - maxBuffer: MAX_COMMAND_OUTPUT_BYTES, - timeout, + maxOutputBytes: MAX_COMMAND_OUTPUT_BYTES, + timeoutMs: timeout, }); -} + return result.stdout; +}); -function atomicWrite(path: string, content: string): void { - const temporary = join(dirname(path), `.${basename(path)}.tmp-${process.pid}`); - writeFileSync(temporary, content, {encoding: 'utf8', flag: 'wx', mode: 0o600}); - renameSync(temporary, path); -} +const atomicWrite = Effect.fn('worktreeReadiness.atomicWrite')(function* (target: string, content: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const system = yield* SystemInfo; + const temporary = path.join(path.dirname(target), `.${path.basename(target)}.tmp-${system.processId}`); + yield* fs.writeFileString(temporary, content, {flag: 'wx', mode: 0o600}); + yield* fs.rename(temporary, target); +}); function parseArguments(arguments_: readonly string[]): BenchmarkOptions { let baselineRef: string | undefined; @@ -499,52 +577,53 @@ function parseArguments(arguments_: readonly string[]): BenchmarkOptions { else if (argument === '--output') outputPath = required(arguments_[++index], argument); else if (argument === '--samples') samples = positiveInteger(arguments_[++index], argument); else if (argument === '--warmups') warmups = nonNegativeInteger(arguments_[++index], argument); - else throw new Error(`Unknown worktree-readiness benchmark option: ${argument}`); + else throw new ScriptError(`Unknown worktree-readiness benchmark option: ${argument}`); } return {baselineRef, candidateRef, outputPath, samples, warmups}; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } function positiveInteger(value: string | undefined, option: string): number { const parsed = nonNegativeInteger(value, option); - if (parsed === 0) throw new Error(`${option} must be at least 1.`); + if (parsed === 0) throw new ScriptError(`${option} must be at least 1.`); return parsed; } function nonNegativeInteger(value: string | undefined, option: string): number { const parsed = Number.parseInt(required(value, option), 10); - if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${option} must be a non-negative integer.`); + if (!Number.isSafeInteger(parsed) || parsed < 0) throw new ScriptError(`${option} must be a non-negative integer.`); return parsed; } function record(value: unknown, label: string): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${label} must be an object.`); + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new ScriptError(`${label} must be an object.`); return value as Record; } function array(value: unknown, label: string): readonly unknown[] { - if (!Array.isArray(value)) throw new Error(`${label} must be an array.`); + if (!Array.isArray(value)) throw new ScriptError(`${label} must be an array.`); return value; } function integerField(record_: Record, key: string): number { const value = record_[key]; - if (!Number.isSafeInteger(value) || Number(value) < 0) throw new Error(`${key} must be a non-negative integer.`); + if (!Number.isSafeInteger(value) || Number(value) < 0) + throw new ScriptError(`${key} must be a non-negative integer.`); return Number(value); } function stringField(record_: Record, key: string): string { const value = record_[key]; - if (typeof value !== 'string' || value.length === 0) throw new Error(`${key} must be a non-empty string.`); + if (typeof value !== 'string' || value.length === 0) throw new ScriptError(`${key} must be a non-empty string.`); return value; } function requireObservation(value: Observation | undefined, label: string): Observation { - if (!value) throw new Error(`Missing ${label} observation.`); + if (!value) throw new ScriptError(`Missing ${label} observation.`); return value; } @@ -552,10 +631,12 @@ function unique(values: readonly string[]): readonly string[] { return [...new Set(values)].sort(); } -function sha256(value: string | Uint8Array): string { - return createHash('sha256').update(value).digest('hex'); +function progress(message: string): Effect.Effect { + return Console.error(`[worktree-readiness] ${message}`); } -function progress(message: string): void { - process.stderr.write(`[worktree-readiness] ${message}\n`); -} +const systemLayer = SystemInfo.layer; +const commandLayer = CommandExecutor.layer.pipe(Layer.provide(systemLayer)); +const WorktreeBenchmarkLayer = Layer.merge(systemLayer, commandLayer).pipe(Layer.provideMerge(BunServices.layer)); + +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(benchmarkWorktreeReadiness, WorktreeBenchmarkLayer)); diff --git a/scripts/build-recall-reranker-dataset.ts b/scripts/build-recall-reranker-dataset.ts index c0df468f..6641669b 100644 --- a/scripts/build-recall-reranker-dataset.ts +++ b/scripts/build-recall-reranker-dataset.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, Layer, Path} from 'effect'; @@ -38,15 +39,15 @@ function parseArguments(args: readonly string[], resolve: (value: string) => str for (let index = 0; index < args.length; index += 1) { const argument = args[index]!; if (argument === '--output') output = resolve(required(args[++index], argument)); - else throw new Error(`Unknown recall reranker dataset option: ${argument}. Pass --help for usage.`); + else throw new ScriptError(`Unknown recall reranker dataset option: ${argument}. Pass --help for usage.`); } return {output}; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } const scriptLayer = Layer.mergeAll(BunServices.layer, SystemInfo.layer); -BunRuntime.runMain(program.pipe(Effect.provide(scriptLayer))); +BunRuntime.runMain(provideScriptLayer(program, scriptLayer)); diff --git a/scripts/build.ts b/scripts/build.ts index 256d72c1..6c5a07e4 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -1,7 +1,8 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Path} from 'effect'; -import {javascriptStringLiteral} from './effect/javascript.js'; +import {javascriptStringLiteral, optionalNativePackageFallbackModule} from './effect/javascript.js'; import {isDevelopmentBuildVersion} from './development-runtime.js'; interface PackageManifest { @@ -27,13 +28,15 @@ const build = Effect.gen(function* () { const configuredDevelopmentVersion = Bun.env.THREADNOTE_DEVELOPMENT_BUILD_VERSION?.trim(); if (configuredDevelopmentVersion && !isDevelopmentBuildVersion(configuredDevelopmentVersion)) { return yield* Effect.fail( - new Error('THREADNOTE_DEVELOPMENT_BUILD_VERSION must be a SHA-bound local development version.'), + new ScriptError('THREADNOTE_DEVELOPMENT_BUILD_VERSION must be a SHA-bound local development version.'), ); } const version = configuredDevelopmentVersion ?? manifest.version; const nativeRuntimeVersion = manifest.dependencies?.[NATIVE_RUNTIME_PACKAGE]; if (!version || !nativeRuntimeVersion || !EXACT_PACKAGE_VERSION.test(nativeRuntimeVersion)) { - return yield* Effect.fail(new Error('package.json must declare version and an exact node-llama-cpp dependency.')); + return yield* Effect.fail( + new ScriptError('package.json must declare version and an exact node-llama-cpp dependency.'), + ); } const target = buildTarget(); @@ -95,7 +98,7 @@ const build = Effect.gen(function* () { const nativePackageRoot = path.join(root, 'node_modules', ...nativePackage.split('/')); if (!(yield* fs.exists(nativePackageRoot))) { return yield* Effect.fail( - new Error(`${nativePackage} is not installed on this target build host. Run bun install before building.`), + new ScriptError(`${nativePackage} is not installed on this target build host. Run bun install before building.`), ); } yield* fs.makeDirectory(nativeRuntimeRoot, {recursive: true}); @@ -139,7 +142,7 @@ function readPackageManifest(fs: FileSystem.FileSystem, path: string) { Effect.flatMap(content => Effect.try({ try: () => JSON.parse(content) as PackageManifest, - catch: cause => new Error('Could not parse package.json.', {cause}), + catch: cause => new ScriptError('Could not parse package.json.', {cause}), }), ), ); @@ -156,26 +159,28 @@ const stageCodeGraphPackageAssets = Effect.fn('build.stageCodeGraphPackageAssets Effect.flatMap(content => Effect.try({ try: () => JSON.parse(content) as unknown, - catch: cause => new Error('Could not parse the code graph asset manifest.', {cause}), + catch: cause => new ScriptError('Could not parse the code graph asset manifest.', {cause}), }), ), ); if (!isRecord(manifest) || !isRecord(manifest.grammars)) { - return yield* Effect.fail(new Error('Code graph asset manifest does not declare grammars.')); + return yield* Effect.fail(new ScriptError('Code graph asset manifest does not declare grammars.')); } for (const [id, value] of Object.entries(manifest.grammars).sort(([left], [right]) => left.localeCompare(right))) { if (!isRecord(value) || typeof value.path !== 'string') { - return yield* Effect.fail(new Error(`Code graph grammar metadata is invalid for ${id}.`)); + return yield* Effect.fail(new ScriptError(`Code graph grammar metadata is invalid for ${id}.`)); } const target = path.join(outputRoot, 'assets', 'code-graph', ...value.path.split('/')); if (!(yield* fs.exists(target))) { if (typeof value.packagePath !== 'string') { - return yield* Effect.fail(new Error(`Code graph grammar ${id} is not vendored and has no package source.`)); + return yield* Effect.fail( + new ScriptError(`Code graph grammar ${id} is not vendored and has no package source.`), + ); } const source = path.join(root, ...value.packagePath.split('/')); if (!(yield* fs.exists(source))) { return yield* Effect.fail( - new Error(`Code graph grammar package source is missing for ${id}: ${value.packagePath}`), + new ScriptError(`Code graph grammar package source is missing for ${id}: ${value.packagePath}`), ); } yield* fs.makeDirectory(path.dirname(target), {recursive: true}); @@ -187,7 +192,9 @@ const stageCodeGraphPackageAssets = Effect.fn('build.stageCodeGraphPackageAssets const licenseSource = path.join(root, ...value.licensePackagePath.split('/')); if (!(yield* fs.exists(licenseSource))) { return yield* Effect.fail( - new Error(`Code graph grammar license package source is missing for ${id}: ${value.licensePackagePath}`), + new ScriptError( + `Code graph grammar license package source is missing for ${id}: ${value.licensePackagePath}`, + ), ); } yield* fs.makeDirectory(path.dirname(licenseTarget), {recursive: true}); @@ -204,13 +211,13 @@ function isRecord(value: unknown): value is Readonly> { function runBuild(options: Bun.BuildConfig) { return Effect.tryPromise({ try: () => Bun.build(options), - catch: cause => new Error('Bun could not build the standalone artifact.', {cause}), + catch: cause => new ScriptError('Bun could not build the standalone artifact.', {cause}), }).pipe( Effect.flatMap(result => result.success ? Effect.void : Effect.fail( - new Error( + new ScriptError( result.logs .map(log => log.message) .filter(Boolean) @@ -250,7 +257,7 @@ function bundleNativeRuntime(entrypoint: string, outfile: string, nativePackage: `const binsDir = Bun.fileURLToPath(new URL('./native', import.meta.url));`, `export const getBinsDir = () => ({binsDir, packageVersion: ${javascriptStringLiteral(nativeRuntimeVersion)}});`, ].join('\n') - : "export const getBinsDir = () => { throw new Error('Optional native package is not included in this Threadnote artifact.'); };", + : optionalNativePackageFallbackModule(), loader: 'js', })); }, @@ -274,7 +281,7 @@ function assertNativeTargetMatchesHost(target: Bun.Build.CompileTarget): void { const architecture = process.arch === 'arm64' ? '(?:arm64|aarch64)' : process.arch; const matchesHost = new RegExp(`^bun-${platform}-${architecture}(?:-|$)`).test(target); if (!matchesHost) { - throw new Error( + throw new ScriptError( `Target ${target} does not match this ${platform}-${process.arch} build host. ` + 'Native local-AI payloads must be assembled on their target OS and architecture.', ); @@ -300,7 +307,7 @@ function nativePackageForTarget(target: Bun.Build.CompileTarget): string { if (target.startsWith('bun-windows-x64')) { return '@node-llama-cpp/win-x64'; } - throw new Error(`No prebuilt native local-AI package is mapped for ${target}.`); + throw new ScriptError(`No prebuilt native local-AI package is mapped for ${target}.`); } -BunRuntime.runMain(build.pipe(Effect.provide(BunServices.layer))); +BunRuntime.runMain(provideScriptLayer(build, BunServices.layer)); diff --git a/scripts/capture-recall-baseline.ts b/scripts/capture-recall-baseline.ts index af92ec83..2303d07f 100644 --- a/scripts/capture-recall-baseline.ts +++ b/scripts/capture-recall-baseline.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect, FileSystem} from 'effect'; import {ApplicationLayer} from '../src/effect/runtime.js'; @@ -14,7 +15,7 @@ const captureBaseline = Effect.gen(function* () { const raw = yield* fs.readFileString(FIXTURE_PATH); const fixture = yield* Effect.try({ try: () => parseRecallEvaluationFixture(JSON.parse(raw)), - catch: cause => new Error(`Could not parse ${FIXTURE_PATH}.`, {cause}), + catch: cause => new ScriptError(`Could not parse ${FIXTURE_PATH}.`, {cause}), }); const result = evaluateRecallFixture(fixture); const artifact = { @@ -48,18 +49,18 @@ function parseArguments(args: readonly string[]): {readonly createdAt: string; r if (argument === '--created-at') { const value = args[++index]; if (!value?.trim() || Number.isNaN(new Date(value).getTime())) { - throw new Error('--created-at requires an ISO timestamp'); + throw new ScriptError('--created-at requires an ISO timestamp'); } createdAt = new Date(value).toISOString(); } else if (argument === '--output') { const value = args[++index]; - if (!value?.trim()) throw new Error('--output requires a path'); + if (!value?.trim()) throw new ScriptError('--output requires a path'); outputPath = value; } else { - throw new Error(`Unknown recall baseline option: ${argument}`); + throw new ScriptError(`Unknown recall baseline option: ${argument}`); } } return {createdAt, outputPath}; } -BunRuntime.runMain(captureBaseline.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(captureBaseline, ApplicationLayer)); diff --git a/scripts/capture-recall-v2-baseline.ts b/scripts/capture-recall-v2-baseline.ts index ca3db3a7..1a4be67b 100644 --- a/scripts/capture-recall-v2-baseline.ts +++ b/scripts/capture-recall-v2-baseline.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect} from 'effect'; import {ApplicationLayer} from '../src/effect/runtime.js'; @@ -67,7 +68,7 @@ function parseArguments(args: readonly string[]): Options { const argument = args[index]!; if (argument === '--created-at') createdAt = isoDate(requiredValue(args[++index], argument)); else if (argument === '--output') outputPath = requiredValue(args[++index], argument); - else throw new Error(`Unknown recall baseline option: ${argument}`); + else throw new ScriptError(`Unknown recall baseline option: ${argument}`); } return {createdAt, outputPath}; } @@ -79,13 +80,13 @@ function sourceDate(): string { function isoDate(value: string): string { const date = new Date(value); - if (Number.isNaN(date.getTime())) throw new Error(`Invalid ISO timestamp: ${value}`); + if (Number.isNaN(date.getTime())) throw new ScriptError(`Invalid ISO timestamp: ${value}`); return date.toISOString(); } function requiredValue(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } -BunRuntime.runMain(captureBaseline.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(captureBaseline, ApplicationLayer)); diff --git a/scripts/check-self-contained.ts b/scripts/check-self-contained.ts index 083fbd5d..7666a01e 100644 --- a/scripts/check-self-contained.ts +++ b/scripts/check-self-contained.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Path} from 'effect'; @@ -106,7 +107,7 @@ const checkSelfContained = Effect.gen(function* () { Effect.flatMap(content => Effect.try({ try: () => JSON.parse(content) as PackageManifest, - catch: cause => new Error('Could not parse package.json.', {cause}), + catch: cause => new ScriptError('Could not parse package.json.', {cause}), }), ), ); @@ -222,7 +223,7 @@ const checkSelfContained = Effect.gen(function* () { } if (failures.length > 0) { - return yield* Effect.fail(new Error(failures.map(failure => `- ${failure}`).join('\n'))); + return yield* Effect.fail(new ScriptError(failures.map(failure => `- ${failure}`).join('\n'))); } yield* Console.log('Self-contained Bun source and release checks passed.'); }); @@ -359,7 +360,7 @@ function parseJsonFile(fs: FileSystem.FileSystem, path: string): Effect.Effect Effect.try({ try: () => JSON.parse(content) as unknown, - catch: cause => new Error(`Could not parse JSON file ${path}.`, {cause}), + catch: cause => new ScriptError(`Could not parse JSON file ${path}.`, {cause}), }), ), Effect.catch(() => Effect.succeed(undefined)), @@ -370,4 +371,4 @@ function isRecord(value: unknown): value is Readonly> { return typeof value === 'object' && value !== null && !Array.isArray(value); } -BunRuntime.runMain(checkSelfContained.pipe(Effect.provide(BunServices.layer))); +BunRuntime.runMain(provideScriptLayer(checkSelfContained, BunServices.layer)); diff --git a/scripts/clean.ts b/scripts/clean.ts index 45e84069..e1ff20f2 100644 --- a/scripts/clean.ts +++ b/scripts/clean.ts @@ -1,6 +1,7 @@ import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Effect, FileSystem, Path} from 'effect'; +import {provideScriptLayer} from './effect/errors.js'; const clean = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -8,4 +9,4 @@ const clean = Effect.gen(function* () { yield* fs.remove(path.resolve(import.meta.dirname, '..', 'dist'), {force: true, recursive: true}); }); -BunRuntime.runMain(clean.pipe(Effect.provide(BunServices.layer))); +BunRuntime.runMain(provideScriptLayer(clean, BunServices.layer)); diff --git a/scripts/code-graph-benchmark-sampler.ts b/scripts/code-graph-benchmark-sampler.ts index 2b8daba6..da68d98e 100644 --- a/scripts/code-graph-benchmark-sampler.ts +++ b/scripts/code-graph-benchmark-sampler.ts @@ -1,10 +1,21 @@ -import {mkdir, readdir, readFile, readlink, realpath, rename, stat, writeFile} from 'fs/promises'; -import {basename, dirname, isAbsolute, join, relative, resolve, sep} from 'path'; +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; +import * as BunRuntime from '@effect/platform-bun/BunRuntime'; +import * as BunServices from '@effect/platform-bun/BunServices'; +import {Clock, Effect, FileSystem, Layer, Option} from 'effect'; +import { + platformPathFor, + runtimeLstat, + runtimePlatform, + runtimeStat, + SystemInfo, + type RuntimeBigIntStats, +} from '../src/effect/system.js'; const MAX_DARWIN_LSOF_BYTES = 8 * 1024 * 1024; const DARWIN_OPEN_FILE_SAMPLE_INTERVAL_MILLISECONDS = 1_000; const MAX_OPEN_FILE_DESCRIPTORS = 65_536; const MAX_OPEN_FILE_PROCESSES = 4_096; +const hostPath = platformPathFor(runtimePlatform); export interface CodeGraphBenchmarkSamplerPhase { readonly cpuMilliseconds?: number; @@ -80,7 +91,8 @@ export interface CodeGraphBenchmarkSamplerCheckpoint { } export function parseCodeGraphBenchmarkSamplerArtifact(value: unknown): CodeGraphBenchmarkSamplerArtifact { - if (typeof value !== 'object' || value === null) throw new Error('Benchmark sampler artifact must be an object.'); + if (typeof value !== 'object' || value === null) + throw new ScriptError('Benchmark sampler artifact must be an object.'); const artifact = value as Partial; if ( (artifact.version !== 2 && artifact.version !== 3 && artifact.version !== 4) || @@ -94,7 +106,7 @@ export function parseCodeGraphBenchmarkSamplerArtifact(value: unknown): CodeGrap typeof artifact.phases !== 'object' || artifact.phases === null ) { - throw new Error('Benchmark sampler artifact is invalid.'); + throw new ScriptError('Benchmark sampler artifact is invalid.'); } const processTelemetry = parseSamplerProcessTelemetry(artifact.platform, artifact.processTelemetry, artifact.version); const temporaryTelemetry = parseSamplerTemporaryTelemetry( @@ -105,17 +117,19 @@ export function parseCodeGraphBenchmarkSamplerArtifact(value: unknown): CodeGrap let phaseSamples = 0; for (const [phase, sample] of Object.entries(artifact.phases)) { if (!phase || !isSamplerPhase(sample, processTelemetry, temporaryTelemetry, artifact.version)) { - throw new Error(`Benchmark sampler phase ${phase || ''} is invalid.`); + throw new ScriptError(`Benchmark sampler phase ${phase || ''} is invalid.`); } phaseSamples += sample.samples; - if (!Number.isSafeInteger(phaseSamples)) throw new Error('Benchmark sampler sample total is invalid.'); + if (!Number.isSafeInteger(phaseSamples)) throw new ScriptError('Benchmark sampler sample total is invalid.'); } - if (phaseSamples !== artifact.samples) throw new Error('Benchmark sampler phase samples do not match its total.'); + if (phaseSamples !== artifact.samples) + throw new ScriptError('Benchmark sampler phase samples do not match its total.'); return artifact as CodeGraphBenchmarkSamplerArtifact; } export function parseCodeGraphBenchmarkSamplerCheckpoint(value: unknown): CodeGraphBenchmarkSamplerCheckpoint { - if (typeof value !== 'object' || value === null) throw new Error('Benchmark sampler checkpoint must be an object.'); + if (typeof value !== 'object' || value === null) + throw new ScriptError('Benchmark sampler checkpoint must be an object.'); const checkpoint = value as Partial; if ( (checkpoint.version !== 2 && checkpoint.version !== 3 && checkpoint.version !== 4) || @@ -123,10 +137,11 @@ export function parseCodeGraphBenchmarkSamplerCheckpoint(value: unknown): CodeGr typeof checkpoint.updatedAt !== 'string' || !Number.isFinite(Date.parse(checkpoint.updatedAt)) ) { - throw new Error('Benchmark sampler checkpoint is invalid.'); + throw new ScriptError('Benchmark sampler checkpoint is invalid.'); } const sampler = parseCodeGraphBenchmarkSamplerArtifact(checkpoint.sampler); - if (sampler.version !== checkpoint.version) throw new Error('Benchmark sampler checkpoint version is inconsistent.'); + if (sampler.version !== checkpoint.version) + throw new ScriptError('Benchmark sampler checkpoint version is inconsistent.'); return checkpoint as CodeGraphBenchmarkSamplerCheckpoint; } @@ -136,7 +151,7 @@ function parseSamplerProcessTelemetry( version: CodeGraphBenchmarkSamplerArtifact['version'], ): CodeGraphBenchmarkSamplerProcessTelemetry { if (typeof value !== 'object' || value === null) { - throw new Error('Benchmark sampler process telemetry must be an object.'); + throw new ScriptError('Benchmark sampler process telemetry must be an object.'); } const telemetry = value as Partial; if (telemetry.availability === 'available') { @@ -183,7 +198,7 @@ function parseSamplerProcessTelemetry( ) { return telemetry as CodeGraphBenchmarkSamplerProcessTelemetry; } - throw new Error('Benchmark sampler process telemetry does not match its platform.'); + throw new ScriptError('Benchmark sampler process telemetry does not match its platform.'); } function parseSamplerTemporaryTelemetry( @@ -192,11 +207,11 @@ function parseSamplerTemporaryTelemetry( version: CodeGraphBenchmarkSamplerArtifact['version'], ): CodeGraphBenchmarkSamplerTemporaryTelemetry | undefined { if (version < 4) { - if (value !== undefined) throw new Error('Legacy benchmark sampler temporary telemetry must be omitted.'); + if (value !== undefined) throw new ScriptError('Legacy benchmark sampler temporary telemetry must be omitted.'); return undefined; } if (typeof value !== 'object' || value === null) { - throw new Error('Benchmark sampler temporary telemetry must be an object.'); + throw new ScriptError('Benchmark sampler temporary telemetry must be an object.'); } const telemetry = value as Partial; if ( @@ -226,7 +241,7 @@ function parseSamplerTemporaryTelemetry( ) { return telemetry as CodeGraphBenchmarkSamplerTemporaryTelemetry; } - throw new Error('Benchmark sampler temporary telemetry does not match its platform.'); + throw new ScriptError('Benchmark sampler temporary telemetry does not match its platform.'); } function isSamplerPhase( @@ -349,25 +364,26 @@ interface SamplerOptions { readonly temporaryRoot: string; } -async function main(): Promise { - const options = parseArguments(process.argv.slice(2)); - const canonicalTemporaryRoot = await canonicalDirectory(options.temporaryRoot); - const clockTicksPerSecond = linuxClockTicksPerSecond(); - const processSampleIntervalMilliseconds = process.platform === 'darwin' ? 250 : options.intervalMilliseconds; +const samplerMain = Effect.gen(function* () { + const system = yield* SystemInfo; + const options = parseArguments(system.processArguments.slice(2)); + const canonicalTemporaryRoot = yield* canonicalDirectory(options.temporaryRoot); + const clockTicksPerSecond = yield* linuxClockTicksPerSecond(); + const processSampleIntervalMilliseconds = system.platform === 'darwin' ? 250 : options.intervalMilliseconds; const openFileSampleIntervalMilliseconds = - process.platform === 'darwin' ? DARWIN_OPEN_FILE_SAMPLE_INTERVAL_MILLISECONDS : options.intervalMilliseconds; - const initialProcessSample = await readProcessTreeSample(options.processId, clockTicksPerSecond); + system.platform === 'darwin' ? DARWIN_OPEN_FILE_SAMPLE_INTERVAL_MILLISECONDS : options.intervalMilliseconds; + const initialProcessSample = yield* readProcessTreeSample(options.processId, clockTicksPerSecond); const processTelemetry = samplerProcessTelemetryContract( - process.platform, + system.platform, initialProcessSample?.rootStartIdentity, processSampleIntervalMilliseconds, ); const parentStartIdentity = initialProcessSample?.rootStartIdentity; const initialTemporaryOpenFiles = initialProcessSample - ? await readOpenTemporaryFileSnapshot(initialProcessSample.processIds, canonicalTemporaryRoot) + ? yield* readOpenTemporaryFileSnapshot(initialProcessSample.processIds, canonicalTemporaryRoot) : undefined; const temporaryTelemetry = samplerTemporaryTelemetryContract( - process.platform, + system.platform, initialTemporaryOpenFiles, openFileSampleIntervalMilliseconds, ); @@ -376,39 +392,43 @@ async function main(): Promise { let pendingInitialProcessSample = previousProcessSample; let pendingInitialTemporaryOpenFiles = temporaryTelemetry.availability === 'available' ? initialTemporaryOpenFiles : undefined; - let lastProcessSampleAt = Date.now(); + let lastProcessSampleAt = yield* Clock.currentTimeMillis; let lastSuccessfulProcessSampleAt = lastProcessSampleAt; - let lastTemporaryOpenSampleAt = Date.now(); + let lastTemporaryOpenSampleAt = lastProcessSampleAt; let samples = 0; let lastCheckpointAt = 0; let readyPublished = false; let stopped: boolean; let stopState: CodeGraphBenchmarkSamplerCheckpoint['state']; do { - const phase = (await readText(options.phasePath))?.trim() || 'unknown'; + const phase = (yield* readText(options.phasePath))?.trim() || 'unknown'; + const sampleStartedAt = yield* Clock.currentTimeMillis; const processSampleDue = pendingInitialProcessSample !== undefined || - Date.now() - lastProcessSampleAt >= processSampleIntervalMilliseconds; + sampleStartedAt - lastProcessSampleAt >= processSampleIntervalMilliseconds; const temporaryOpenSampleDue = pendingInitialTemporaryOpenFiles !== undefined || - Date.now() - lastTemporaryOpenSampleAt >= openFileSampleIntervalMilliseconds; + sampleStartedAt - lastTemporaryOpenSampleAt >= openFileSampleIntervalMilliseconds; const [databaseBytes, walBytes, shmBytes, journalBytes, temporaryLinkedFiles, observedProcessSample] = - await Promise.all([ - fileBytes(options.databasePath), - fileBytes(`${options.databasePath}-wal`), - fileBytes(`${options.databasePath}-shm`), - fileBytes(`${options.databasePath}-journal`), - directoryFileSnapshot(canonicalTemporaryRoot), - processSampleDue - ? pendingInitialProcessSample !== undefined - ? Promise.resolve(pendingInitialProcessSample) - : readProcessTreeSample(options.processId, clockTicksPerSecond) - : Promise.resolve(undefined), - ]); + yield* Effect.all( + [ + fileBytes(options.databasePath), + fileBytes(`${options.databasePath}-wal`), + fileBytes(`${options.databasePath}-shm`), + fileBytes(`${options.databasePath}-journal`), + directoryFileSnapshot(canonicalTemporaryRoot), + processSampleDue + ? pendingInitialProcessSample !== undefined + ? Effect.succeed(pendingInitialProcessSample) + : readProcessTreeSample(options.processId, clockTicksPerSecond) + : Effect.succeed(undefined), + ], + {concurrency: 'unbounded'}, + ); const processSample = processSampleDue ? observedProcessSample : undefined; if (processSampleDue) { pendingInitialProcessSample = undefined; - lastProcessSampleAt = Date.now(); + lastProcessSampleAt = yield* Clock.currentTimeMillis; } const processForOpenSample = processSample ?? previousProcessSample; const temporaryOpenFiles = @@ -416,15 +436,15 @@ async function main(): Promise { ? pendingInitialTemporaryOpenFiles !== undefined ? pendingInitialTemporaryOpenFiles : processForOpenSample - ? await readOpenTemporaryFileSnapshot(processForOpenSample.processIds, canonicalTemporaryRoot) + ? yield* readOpenTemporaryFileSnapshot(processForOpenSample.processIds, canonicalTemporaryRoot) : undefined : undefined; if (temporaryOpenSampleDue) { pendingInitialTemporaryOpenFiles = undefined; - lastTemporaryOpenSampleAt = Date.now(); + lastTemporaryOpenSampleAt = yield* Clock.currentTimeMillis; } const temporaryBytes = mergeTemporaryFileSnapshots(temporaryLinkedFiles, temporaryOpenFiles).bytes; - const parentExists = processExists(options.processId); + const parentExists = system.isProcessRunning(options.processId); const parentExited = samplerParentExited(parentStartIdentity, processSample?.rootStartIdentity, parentExists); const telemetrySample = processTelemetry.availability === 'available' && !parentExited ? processSample : undefined; const current = phases.get(phase) ?? { @@ -463,7 +483,7 @@ async function main(): Promise { } current.temporaryPeakBytes = Math.max(current.temporaryPeakBytes, temporaryBytes); if (processTelemetry.availability === 'available' && processSampleDue) { - const observedAt = Date.now(); + const observedAt = yield* Clock.currentTimeMillis; current.processSampleAttempts += 1; current.processSampleGapPeakMilliseconds = Math.max( current.processSampleGapPeakMilliseconds, @@ -485,33 +505,48 @@ async function main(): Promise { current.samples += 1; samples += 1; phases.set(phase, current); - const requestedStop = await readText(options.stopPath); + const requestedStop = yield* readText(options.stopPath); stopped = requestedStop !== undefined || parentExited; stopState = requestedStop !== undefined ? parseStopState(requestedStop) : parentExited ? 'parent-exited' : 'running'; - const now = Date.now(); + const now = yield* Clock.currentTimeMillis; if (stopped || samples === 1 || now - lastCheckpointAt >= options.checkpointIntervalMilliseconds) { - await writeCheckpoint( + yield* writeCheckpoint( options.checkpointPath, - samplerArtifact(options.intervalMilliseconds, phases, processTelemetry, temporaryTelemetry, samples), + samplerArtifact( + options.intervalMilliseconds, + phases, + system.platform, + processTelemetry, + temporaryTelemetry, + samples, + ), stopState, ); if (!readyPublished) { - await atomicWriteFile(options.readyPath, '{"checkpointVersion":4,"version":1}\n'); + yield* atomicWriteFile(options.readyPath, '{"checkpointVersion":4,"version":1}\n'); readyPublished = true; } lastCheckpointAt = now; } - if (!stopped) await Bun.sleep(options.intervalMilliseconds); + if (!stopped) yield* Effect.sleep(options.intervalMilliseconds); } while (!stopped); - const artifact = samplerArtifact(options.intervalMilliseconds, phases, processTelemetry, temporaryTelemetry, samples); - await atomicWriteFile(options.outputPath, `${JSON.stringify(artifact)}\n`); -} + const artifact = samplerArtifact( + options.intervalMilliseconds, + phases, + system.platform, + processTelemetry, + temporaryTelemetry, + samples, + ); + yield* atomicWriteFile(options.outputPath, `${JSON.stringify(artifact)}\n`); +}); function samplerArtifact( intervalMilliseconds: number, phases: ReadonlyMap, + platform: string, processTelemetry: CodeGraphBenchmarkSamplerProcessTelemetry, temporaryTelemetry: CodeGraphBenchmarkSamplerTemporaryTelemetry, samples: number, @@ -562,7 +597,7 @@ function samplerArtifact( ]; }), ), - platform: process.platform, + platform, processTelemetry, samples, temporaryTelemetry, @@ -570,26 +605,31 @@ function samplerArtifact( }; } -async function writeCheckpoint( +const writeCheckpoint = Effect.fn('codeGraphBenchmarkSampler.writeCheckpoint')(function* ( checkpointPath: string, sampler: CodeGraphBenchmarkSamplerArtifact, state: CodeGraphBenchmarkSamplerCheckpoint['state'], -): Promise { +) { const checkpoint: CodeGraphBenchmarkSamplerCheckpoint = { sampler, state, - updatedAt: new Date().toISOString(), + updatedAt: new Date(yield* Clock.currentTimeMillis).toISOString(), version: sampler.version, }; - await atomicWriteFile(checkpointPath, `${JSON.stringify(checkpoint)}\n`); -} - -async function atomicWriteFile(outputPath: string, contents: string): Promise { - await mkdir(dirname(outputPath), {recursive: true, mode: 0o700}); - const temporaryPath = `${outputPath}.${process.pid}.tmp`; - await writeFile(temporaryPath, contents, {encoding: 'utf8', mode: 0o600}); - await rename(temporaryPath, outputPath); -} + yield* atomicWriteFile(checkpointPath, `${JSON.stringify(checkpoint)}\n`); +}); + +const atomicWriteFile = Effect.fn('codeGraphBenchmarkSampler.atomicWriteFile')(function* ( + outputPath: string, + contents: string, +) { + const fs = yield* FileSystem.FileSystem; + const system = yield* SystemInfo; + yield* fs.makeDirectory(hostPath.dirname(outputPath), {recursive: true, mode: 0o700}); + const temporaryPath = `${outputPath}.${system.processId}.tmp`; + yield* fs.writeFileString(temporaryPath, contents, {mode: 0o600}); + yield* fs.rename(temporaryPath, outputPath); +}); function parseStopState(value: string): 'aborted' | 'complete' { return value.trim() === 'complete' ? 'complete' : 'aborted'; @@ -708,25 +748,23 @@ export function processTreeDelta( return {cpuMilliseconds, ioReadBytes, ioWriteBytes}; } -async function readProcessTreeSample( +const readProcessTreeSample = Effect.fn('codeGraphBenchmarkSampler.readProcessTreeSample')(function* ( processId: number, clockTicksPerSecond: number, -): Promise { - if (process.platform === 'linux') return readLinuxProcessTreeSample(processId, clockTicksPerSecond, process.pid); - if (process.platform === 'darwin') return readDarwinProcessTreeSample(processId, process.pid); +) { + const system = yield* SystemInfo; + if (system.platform === 'linux') { + return yield* readLinuxProcessTreeSample(processId, clockTicksPerSecond, system.processId); + } + if (system.platform === 'darwin') return yield* readDarwinProcessTreeSample(processId, system.processId); return undefined; -} - -interface LinuxProcessEntryRead { - readonly childProcessIds: readonly number[]; - readonly entry: BenchmarkProcessTreeEntry; -} +}); -async function readLinuxProcessTreeSample( +const readLinuxProcessTreeSample = Effect.fn('codeGraphBenchmarkSampler.readLinuxProcessTreeSample')(function* ( rootProcessId: number, clockTicksPerSecond: number, excludedProcessId: number, -): Promise { +) { const pending: Array<{readonly expectedParent?: number; readonly processId: number}> = [{processId: rootProcessId}]; const entries: BenchmarkProcessTreeEntry[] = []; const visited = new Set(); @@ -734,7 +772,7 @@ async function readLinuxProcessTreeSample( const next = pending.shift(); if (!next || visited.has(next.processId)) continue; visited.add(next.processId); - const observed = await readLinuxProcessEntry(next.processId, clockTicksPerSecond); + const observed = yield* readLinuxProcessEntry(next.processId, clockTicksPerSecond); if (!observed || (next.expectedParent !== undefined && observed.entry.parentProcessId !== next.expectedParent)) { continue; } @@ -742,22 +780,26 @@ async function readLinuxProcessTreeSample( pending.push(...observed.childProcessIds.map(processId => ({expectedParent: observed.entry.processId, processId}))); } return aggregateProcessTree(entries, rootProcessId, undefined, excludedProcessId); -} +}); -async function readLinuxProcessEntry( +const readLinuxProcessEntry = Effect.fn('codeGraphBenchmarkSampler.readLinuxProcessEntry')(function* ( processId: number, clockTicksPerSecond: number, -): Promise { - try { - const firstStatText = await readFile(`/proc/${processId}/stat`, 'utf8'); +) { + const fs = yield* FileSystem.FileSystem; + return yield* Effect.gen(function* () { + const firstStatText = yield* fs.readFileString(`/proc/${processId}/stat`); const firstStat = parseLinuxProcessStat(firstStatText); if (!firstStat) return undefined; - const [statusText, ioText, childrenText, validatedStatText] = await Promise.all([ - readFile(`/proc/${processId}/status`, 'utf8'), - readText(`/proc/${processId}/io`), - readText(`/proc/${processId}/task/${processId}/children`), - readFile(`/proc/${processId}/stat`, 'utf8'), - ]); + const [statusText, ioText, childrenText, validatedStatText] = yield* Effect.all( + [ + fs.readFileString(`/proc/${processId}/status`), + readText(`/proc/${processId}/io`), + readText(`/proc/${processId}/task/${processId}/children`), + fs.readFileString(`/proc/${processId}/stat`), + ], + {concurrency: 'unbounded'}, + ); const validatedStat = parseLinuxProcessStat(validatedStatText); if (!validatedStat || validatedStat.startIdentity !== firstStat.startIdentity) return undefined; const rssMatch = /^VmRSS:\s+(\d+)\s+kB$/m.exec(statusText); @@ -776,10 +818,8 @@ async function readLinuxProcessEntry( startIdentity: validatedStat.startIdentity, }, }; - } catch { - return undefined; - } -} + }).pipe(Effect.option, Effect.map(Option.getOrUndefined)); +}); function parseLinuxChildProcessIds(text: string): readonly number[] { return text @@ -801,27 +841,28 @@ export function parseLinuxProcessIo( return {readBytes, writeBytes}; } -async function readDarwinProcessTreeSample( +const readDarwinProcessTreeSample = Effect.fn('codeGraphBenchmarkSampler.readDarwinProcessTreeSample')(function* ( rootProcessId: number, excludedProcessId: number, -): Promise { - try { - const result = Bun.spawnSync({ - cmd: ['/bin/ps', '-axo', 'pid=,ppid=,lstart=,time=,rss='], - env: {LC_ALL: 'C'}, - stderr: 'ignore', - }); - if (result.exitCode !== 0) return undefined; - return aggregateProcessTree( - parseDarwinProcessList(new TextDecoder().decode(result.stdout)), - rootProcessId, - undefined, - excludedProcessId, - ); - } catch { - return undefined; - } -} +) { + return yield* Effect.try({ + try: () => { + const result = Bun.spawnSync({ + cmd: ['/bin/ps', '-axo', 'pid=,ppid=,lstart=,time=,rss='], + env: {LC_ALL: 'C'}, + stderr: 'ignore', + }); + if (result.exitCode !== 0) return undefined; + return aggregateProcessTree( + parseDarwinProcessList(new TextDecoder().decode(result.stdout)), + rootProcessId, + undefined, + excludedProcessId, + ); + }, + catch: scriptError, + }).pipe(Effect.option, Effect.map(Option.getOrUndefined)); +}); export function parseDarwinProcessList(output: string): readonly BenchmarkProcessTreeEntry[] { return output @@ -1021,12 +1062,13 @@ export function mergeTemporaryFileSnapshots( export function isOpenTemporaryFilePath(target: string, canonicalTemporaryRoot: string): boolean { const withoutDeletedMarker = target.endsWith(' (deleted)') ? target.slice(0, -' (deleted)'.length) : target; - if (!isAbsolute(withoutDeletedMarker)) return false; - const candidate = resolve(withoutDeletedMarker); - const relativePath = relative(canonicalTemporaryRoot, candidate); + if (!hostPath.isAbsolute(withoutDeletedMarker)) return false; + const candidate = hostPath.resolve(withoutDeletedMarker); + const relativePath = hostPath.relative(canonicalTemporaryRoot, candidate); const belongsToTemporaryRoot = - relativePath === '' || (relativePath !== '..' && !relativePath.startsWith(`..${sep}`) && !isAbsolute(relativePath)); - return belongsToTemporaryRoot || /^etilqs_[a-z0-9]+$/i.test(basename(candidate)); + relativePath === '' || + (relativePath !== '..' && !relativePath.startsWith(`..${hostPath.sep}`) && !hostPath.isAbsolute(relativePath)); + return belongsToTemporaryRoot || /^etilqs_[a-z0-9]+$/i.test(hostPath.basename(candidate)); } export function parseDarwinOpenFileList( @@ -1112,31 +1154,31 @@ export function parseDarwinOpenFileList( return observedRoot ? temporaryFileSnapshot(files) : undefined; } -async function readOpenTemporaryFileSnapshot( +const readOpenTemporaryFileSnapshot = Effect.fn('codeGraphBenchmarkSampler.readOpenTemporaryFileSnapshot')(function* ( processIds: readonly number[], canonicalTemporaryRoot: string, -): Promise { - if (process.platform === 'linux') return readLinuxOpenTemporaryFiles(processIds, canonicalTemporaryRoot); - if (process.platform === 'darwin') return readDarwinOpenTemporaryFiles(processIds, canonicalTemporaryRoot); +) { + const system = yield* SystemInfo; + if (system.platform === 'linux') return yield* readLinuxOpenTemporaryFiles(processIds, canonicalTemporaryRoot); + if (system.platform === 'darwin') return yield* readDarwinOpenTemporaryFiles(processIds, canonicalTemporaryRoot); return undefined; -} +}); -async function readLinuxOpenTemporaryFiles( +const readLinuxOpenTemporaryFiles = Effect.fn('codeGraphBenchmarkSampler.readLinuxOpenTemporaryFiles')(function* ( processIds: readonly number[], canonicalTemporaryRoot: string, -): Promise { +) { + const fs = yield* FileSystem.FileSystem; const rootProcessId = processIds[0]; if (rootProcessId === undefined || processIds.length > MAX_OPEN_FILE_PROCESSES) return undefined; const descriptors: Array<{readonly descriptorPath: string; readonly processId: number}> = []; for (const processId of [...new Set(processIds)]) { - let names: readonly string[]; - try { - names = await readdir(`/proc/${processId}/fd`); - } catch { + const names = yield* fs.readDirectory(`/proc/${processId}/fd`).pipe(Effect.option); + if (Option.isNone(names)) { if (processId === rootProcessId) return undefined; continue; } - for (const name of names) { + for (const name of names.value) { if (!/^\d+$/.test(name)) continue; if (descriptors.length >= MAX_OPEN_FILE_DESCRIPTORS) return undefined; descriptors.push({descriptorPath: `/proc/${processId}/fd/${name}`, processId}); @@ -1144,65 +1186,67 @@ async function readLinuxOpenTemporaryFiles( } const files = new Map(); for (let offset = 0; offset < descriptors.length; offset += 32) { - const observed = await Promise.all( - descriptors - .slice(offset, offset + 32) - .map(({descriptorPath}) => readLinuxOpenTemporaryFile(descriptorPath, canonicalTemporaryRoot)), + const observed = yield* Effect.forEach( + descriptors.slice(offset, offset + 32), + ({descriptorPath}) => readLinuxOpenTemporaryFile(descriptorPath, canonicalTemporaryRoot), + {concurrency: 'unbounded'}, ); for (const file of observed) { if (file) files.set(file.identity, Math.max(files.get(file.identity) ?? 0, file.bytes)); } } return temporaryFileSnapshot(files); -} +}); -async function readLinuxOpenTemporaryFile( +const readLinuxOpenTemporaryFile = Effect.fn('codeGraphBenchmarkSampler.readLinuxOpenTemporaryFile')(function* ( descriptorPath: string, canonicalTemporaryRoot: string, -): Promise<{readonly bytes: number; readonly identity: string} | undefined> { - try { - const [target, info] = await Promise.all([readlink(descriptorPath), stat(descriptorPath, {bigint: true})]); - if (!info.isFile() || !isOpenTemporaryFilePath(target, canonicalTemporaryRoot)) return undefined; - const bytes = safeBigIntByteCount(info.size); - return bytes === undefined ? undefined : {bytes, identity: `${info.dev}:${info.ino}`}; - } catch { - return undefined; - } -} +) { + const fs = yield* FileSystem.FileSystem; + return yield* Effect.gen(function* () { + const [target, info] = yield* Effect.all( + [fs.readLink(descriptorPath), Effect.tryPromise({try: () => runtimeStat(descriptorPath), catch: scriptError})], + {concurrency: 2}, + ); + if (!isOpenTemporaryFilePath(target, canonicalTemporaryRoot)) return undefined; + return temporaryFileObservationFromStats(info); + }).pipe(Effect.option, Effect.map(Option.getOrUndefined)); +}); -async function readDarwinOpenTemporaryFiles( +const readDarwinOpenTemporaryFiles = Effect.fn('codeGraphBenchmarkSampler.readDarwinOpenTemporaryFiles')(function* ( processIds: readonly number[], canonicalTemporaryRoot: string, -): Promise { +) { const uniqueProcessIds = [...new Set(processIds)].filter( processId => Number.isSafeInteger(processId) && processId > 0, ); const rootProcessId = uniqueProcessIds[0]; if (rootProcessId === undefined || uniqueProcessIds.length > MAX_OPEN_FILE_PROCESSES) return undefined; - try { - const result = Bun.spawnSync({ - cmd: ['/usr/sbin/lsof', '-nP', '-a', '-p', uniqueProcessIds.join(','), '-d', '0-1048575', '-F0pftsiDn'], - env: {LC_ALL: 'C'}, - maxBuffer: MAX_DARWIN_LSOF_BYTES, - stderr: 'ignore', - }); - if ( - (result.signalCode !== undefined && result.signalCode !== null) || - ![0, 1].includes(result.exitCode) || - result.stdout.byteLength >= MAX_DARWIN_LSOF_BYTES - ) { - return undefined; - } - return parseDarwinOpenFileList( - new TextDecoder().decode(result.stdout), - uniqueProcessIds, - rootProcessId, - canonicalTemporaryRoot, - ); - } catch { - return undefined; - } -} + return yield* Effect.try({ + try: () => { + const result = Bun.spawnSync({ + cmd: ['/usr/sbin/lsof', '-nP', '-a', '-p', uniqueProcessIds.join(','), '-d', '0-1048575', '-F0pftsiDn'], + env: {LC_ALL: 'C'}, + maxBuffer: MAX_DARWIN_LSOF_BYTES, + stderr: 'ignore', + }); + if ( + (result.signalCode !== undefined && result.signalCode !== null) || + ![0, 1].includes(result.exitCode) || + result.stdout.byteLength >= MAX_DARWIN_LSOF_BYTES + ) { + return undefined; + } + return parseDarwinOpenFileList( + new TextDecoder().decode(result.stdout), + uniqueProcessIds, + rootProcessId, + canonicalTemporaryRoot, + ); + }, + catch: scriptError, + }).pipe(Effect.option, Effect.map(Option.getOrUndefined)); +}); function parseDarwinDevice(value: string | undefined): string | undefined { if (value === undefined || !/^(?:0x[0-9a-f]+|\d+)$/i.test(value)) return undefined; @@ -1222,6 +1266,14 @@ function parseSafeByteCount(value: string | undefined): number | undefined { } } +export function temporaryFileObservationFromStats( + info: Pick, +): {readonly bytes: number; readonly identity: string} | undefined { + if (!info.isFile()) return undefined; + const bytes = safeBigIntByteCount(info.size); + return bytes === undefined ? undefined : {bytes, identity: `${info.dev}:${info.ino}`}; +} + function safeBigIntByteCount(value: bigint): number | undefined { return value >= 0n && value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : undefined; } @@ -1253,124 +1305,102 @@ export function samplerParentExited( ); } -function linuxClockTicksPerSecond(): number { - if (process.platform !== 'linux') return 100; - try { - const result = Bun.spawnSync({cmd: ['getconf', 'CLK_TCK'], stderr: 'ignore'}); - const value = Number(new TextDecoder().decode(result.stdout).trim()); - return Number.isFinite(value) && value > 0 ? value : 100; - } catch { - return 100; - } -} - -async function canonicalDirectory(directory: string): Promise { - try { - return await realpath(directory); - } catch { - return resolve(directory); - } -} - -async function directoryFileSnapshot(directory: string): Promise { +const linuxClockTicksPerSecond = Effect.fn('codeGraphBenchmarkSampler.linuxClockTicksPerSecond')(function* () { + const system = yield* SystemInfo; + if (system.platform !== 'linux') return 100; + return yield* Effect.try({ + try: () => { + const result = Bun.spawnSync({cmd: ['getconf', 'CLK_TCK'], stderr: 'ignore'}); + const value = Number(new TextDecoder().decode(result.stdout).trim()); + return Number.isFinite(value) && value > 0 ? value : 100; + }, + catch: scriptError, + }).pipe(Effect.catch(() => Effect.succeed(100))); +}); + +const canonicalDirectory = Effect.fn('codeGraphBenchmarkSampler.canonicalDirectory')(function* (directory: string) { + const fs = yield* FileSystem.FileSystem; + return yield* fs.realPath(directory).pipe(Effect.catch(() => Effect.succeed(hostPath.resolve(directory)))); +}); + +const directoryFileSnapshot = Effect.fn('codeGraphBenchmarkSampler.directoryFileSnapshot')(function* ( + directory: string, +) { const files = new Map(); - await collectDirectoryFiles(directory, files); + yield* collectDirectoryFiles(directory, files); return temporaryFileSnapshot(files); -} - -async function collectDirectoryFiles(directory: string, files: Map): Promise { - try { - for (const entry of await readdir(directory, {withFileTypes: true})) { - const child = join(directory, entry.name); - if (entry.isDirectory()) { - await collectDirectoryFiles(child, files); - } else if (entry.isFile()) { - const observed = await fileSnapshotEntry(child); - if (observed) files.set(observed.identity, Math.max(files.get(observed.identity) ?? 0, observed.bytes)); +}); + +function collectDirectoryFiles( + directory: string, + files: Map, +): Effect.Effect { + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const names = yield* fs.readDirectory(directory).pipe(Effect.option); + if (Option.isNone(names)) return; + for (const name of names.value) { + const child = hostPath.join(directory, name); + const info = yield* Effect.tryPromise({try: () => runtimeLstat(child), catch: scriptError}).pipe(Effect.option); + if (Option.isNone(info)) continue; + if (info.value.isDirectory()) { + yield* collectDirectoryFiles(child, files); + continue; } + const observed = temporaryFileObservationFromStats(info.value); + if (observed) files.set(observed.identity, Math.max(files.get(observed.identity) ?? 0, observed.bytes)); } - } catch { - return; - } -} - -async function fileSnapshotEntry( - file: string, -): Promise<{readonly bytes: number; readonly identity: string} | undefined> { - try { - const info = await stat(file, {bigint: true}); - if (!info.isFile()) return undefined; - const bytes = safeBigIntByteCount(info.size); - return bytes === undefined ? undefined : {bytes, identity: `${info.dev}:${info.ino}`}; - } catch { - return undefined; - } -} - -async function fileBytes(file: string): Promise { - try { - const info = await stat(file); - return info.isFile() ? info.size : 0; - } catch { - return 0; - } + }); } -function processExists(processId: number): boolean { - try { - process.kill(processId, 0); - return true; - } catch (cause: unknown) { - return !( - typeof cause === 'object' && - cause !== null && - 'code' in cause && - (cause as {readonly code?: unknown}).code === 'ESRCH' - ); - } -} +const fileBytes = Effect.fn('codeGraphBenchmarkSampler.fileBytes')(function* (file: string) { + const fs = yield* FileSystem.FileSystem; + const info = yield* fs.stat(file).pipe(Effect.option); + if (Option.isNone(info) || info.value.type !== 'File') return 0; + return safeBigIntByteCount(info.value.size) ?? 0; +}); -async function readText(file: string): Promise { - try { - return await readFile(file, 'utf8'); - } catch { - return undefined; - } -} +const readText = Effect.fn('codeGraphBenchmarkSampler.readText')(function* (file: string) { + const fs = yield* FileSystem.FileSystem; + return yield* fs.readFileString(file).pipe(Effect.option, Effect.map(Option.getOrUndefined)); +}); function parseArguments(args: readonly string[]): SamplerOptions { const values = new Map(); for (let index = 0; index < args.length; index += 2) { const flag = args[index]; const value = args[index + 1]; - if (!flag?.startsWith('--') || value === undefined) throw new Error(`Invalid benchmark sampler argument ${flag}.`); + if (!flag?.startsWith('--') || value === undefined) + throw new ScriptError(`Invalid benchmark sampler argument ${flag}.`); values.set(flag, value); } const required = (flag: string) => { const value = values.get(flag); - if (!value) throw new Error(`Missing benchmark sampler argument ${flag}.`); + if (!value) throw new ScriptError(`Missing benchmark sampler argument ${flag}.`); return value; }; const processId = Number(required('--pid')); const intervalMilliseconds = Number(required('--interval-ms')); const checkpointIntervalMilliseconds = Number(required('--checkpoint-ms')); - if (!Number.isSafeInteger(processId) || processId <= 0) throw new Error('Sampler PID must be positive.'); + if (!Number.isSafeInteger(processId) || processId <= 0) throw new ScriptError('Sampler PID must be positive.'); if (!Number.isSafeInteger(intervalMilliseconds) || intervalMilliseconds < 10) { - throw new Error('Sampler interval must be at least 10 milliseconds.'); + throw new ScriptError('Sampler interval must be at least 10 milliseconds.'); } if (!Number.isSafeInteger(checkpointIntervalMilliseconds) || checkpointIntervalMilliseconds < intervalMilliseconds) { - throw new Error('Sampler checkpoint interval must be at least the sampling interval.'); + throw new ScriptError('Sampler checkpoint interval must be at least the sampling interval.'); } const outputPath = required('--output'); const checkpointPath = required('--checkpoint-output'); const readyPath = required('--ready'); - if (dirname(outputPath) === outputPath) throw new Error('Sampler output path must have a parent directory.'); - if (dirname(checkpointPath) === checkpointPath) { - throw new Error('Sampler checkpoint path must have a parent directory.'); + if (hostPath.dirname(outputPath) === outputPath) + throw new ScriptError('Sampler output path must have a parent directory.'); + if (hostPath.dirname(checkpointPath) === checkpointPath) { + throw new ScriptError('Sampler checkpoint path must have a parent directory.'); } - if (dirname(readyPath) === readyPath) throw new Error('Sampler ready path must have a parent directory.'); + if (hostPath.dirname(readyPath) === readyPath) + throw new ScriptError('Sampler ready path must have a parent directory.'); if (new Set([checkpointPath, outputPath, readyPath]).size !== 3) { - throw new Error('Sampler checkpoint, output, and ready paths must be distinct.'); + throw new ScriptError('Sampler checkpoint, output, and ready paths must be distinct.'); } return { checkpointIntervalMilliseconds, @@ -1386,4 +1416,6 @@ function parseArguments(args: readonly string[]): SamplerOptions { }; } -if (import.meta.main) await main(); +const SamplerLayer = SystemInfo.layer.pipe(Layer.provideMerge(BunServices.layer)); + +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(samplerMain, SamplerLayer)); diff --git a/scripts/code-graph-fixture.ts b/scripts/code-graph-fixture.ts index 19e14b16..7e8363b4 100644 --- a/scripts/code-graph-fixture.ts +++ b/scripts/code-graph-fixture.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {Effect, FileSystem, Path, Schedule} from 'effect'; import { CODE_GRAPH_GENERIC_JSON_EXCLUSION_BYTES, @@ -133,7 +134,7 @@ export const prepareCodeGraphFixture = Effect.fn('codeGraphFixture.prepare')(fun const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (!/^code-graph-[a-z0-9-]+$/.test(fixture)) { - return yield* Effect.fail(new Error(`Invalid code graph fixture name: ${fixture}.`)); + return yield* Effect.fail(new ScriptError(`Invalid code graph fixture name: ${fixture}.`)); } const source = yield* path.fromFileUrl( new URL(`../test/evaluation/fixtures/${fixture}/repository/`, import.meta.url), @@ -164,7 +165,7 @@ export const prepareGeneratedCodeGraphFixture = Effect.fn('codeGraphFixture.prep const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (!Number.isSafeInteger(targetSymbols) || targetSymbols < 1) { - return yield* Effect.fail(new Error('Generated code graph target must be a positive safe integer.')); + return yield* Effect.fail(new ScriptError('Generated code graph target must be a positive safe integer.')); } const root = yield* makeOwnedTempDirectoryScoped('threadnote-code-graph-scale-'); const repository = path.join(root, 'repository'); @@ -450,10 +451,12 @@ export function productionSymbolName(index: number, workspaceIndex: number): str export function productionWorkspaceRoots(count: number, activeExcludedCount = 0): readonly string[] { if (!Number.isSafeInteger(count) || count < 1) { - throw new Error('Production code graph workspace count must be a positive safe integer.'); + throw new ScriptError('Production code graph workspace count must be a positive safe integer.'); } if (!Number.isSafeInteger(activeExcludedCount) || activeExcludedCount < 0 || activeExcludedCount >= count) { - throw new Error('Production code graph active workspace-excluded package count must leave one included package.'); + throw new ScriptError( + 'Production code graph active workspace-excluded package count must leave one included package.', + ); } const includedCount = count - activeExcludedCount; const fixedCandidates = [ @@ -510,17 +513,17 @@ export function validateProductionProfile( }; for (const [name, value] of Object.entries(scalarCounts)) { if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`Production code graph profile ${name} must be a positive safe integer.`); + throw new ScriptError(`Production code graph profile ${name} must be a positive safe integer.`); } } for (const [name, value] of Object.entries(profile.classMix)) { if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`Production code graph profile class mix ${name} must be a positive safe integer.`); + throw new ScriptError(`Production code graph profile class mix ${name} must be a positive safe integer.`); } } for (const [name, value] of Object.entries(profile.duplicateBlobs)) { if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`Production code graph profile duplicate blob ${name} must be a positive safe integer.`); + throw new ScriptError(`Production code graph profile duplicate blob ${name} must be a positive safe integer.`); } } if ( @@ -528,58 +531,66 @@ export function validateProductionProfile( profile.version !== 2 || profile.surrogate !== 'threadnote-4.1.0-beta.1-public-monorepo' ) { - throw new Error('Unsupported production code graph fixture profile.'); + throw new ScriptError('Unsupported production code graph fixture profile.'); } if (profile.worktreeChurnScenarioCount !== PRODUCTION_WORKTREE_CHURN_SCENARIOS.length) { - throw new Error('Production code graph fixture must declare the reviewed six-scenario worktree churn matrix.'); + throw new ScriptError( + 'Production code graph fixture must declare the reviewed six-scenario worktree churn matrix.', + ); } if (profile.declarationSymbols < profile.sourceFiles) { - throw new Error('Production code graph fixture requires at least one declaration per source file.'); + throw new ScriptError('Production code graph fixture requires at least one declaration per source file.'); } if (profile.classMix.typescriptSourceFiles + profile.classMix.tsxSourceFiles !== profile.sourceFiles) { - throw new Error('Production code graph source class counts must equal sourceFiles.'); + throw new ScriptError('Production code graph source class counts must equal sourceFiles.'); } if (productionRepositoryFileCount(profile.classMix) !== profile.targetRepositoryFiles) { - throw new Error('Production code graph class mix must equal targetRepositoryFiles.'); + throw new ScriptError('Production code graph class mix must equal targetRepositoryFiles.'); } if (productionEligibleFileCount(profile.classMix) !== profile.targetEligibleFiles) { - throw new Error('Production code graph eligible class mix must equal targetEligibleFiles.'); + throw new ScriptError('Production code graph eligible class mix must equal targetEligibleFiles.'); } if (profile.classMix.packageManifestFiles !== profile.workspaceCount + 1) { - throw new Error('Production code graph package manifest count must cover the root and every package.'); + throw new ScriptError('Production code graph package manifest count must cover the root and every package.'); } if (profile.classMix.workspaceManifestFiles !== 1) { - throw new Error('Production code graph fixture requires exactly one pnpm workspace manifest.'); + throw new ScriptError('Production code graph fixture requires exactly one pnpm workspace manifest.'); } if (profile.classMix.tsconfigFiles > profile.workspaceCount + 1) { - throw new Error('Production code graph tsconfig count exceeds the root and package count.'); + throw new ScriptError('Production code graph tsconfig count exceeds the root and package count.'); } if (profile.classMix.nxProjectFiles > profile.workspaceCount) { - throw new Error('Production code graph Nx project count exceeds the package count.'); + throw new ScriptError('Production code graph Nx project count exceeds the package count.'); } if (profile.activeWorkspaceExcludedSourceFiles >= profile.sourceFiles) { - throw new Error('Production code graph active workspace-excluded source count must leave included sourceFiles.'); + throw new ScriptError( + 'Production code graph active workspace-excluded source count must leave included sourceFiles.', + ); } if (profile.duplicateBlobs.generatedSvgVariants > profile.classMix.generatedSvgFiles) { - throw new Error('Production code graph generated SVG variants exceed generated SVG files.'); + throw new ScriptError('Production code graph generated SVG variants exceed generated SVG files.'); } if (profile.duplicateBlobs.heavyJsonVariants > profile.classMix.duplicateHeavyJsonFiles) { - throw new Error('Production code graph heavy JSON variants exceed duplicate heavy JSON files.'); + throw new ScriptError('Production code graph heavy JSON variants exceed duplicate heavy JSON files.'); } if (profile.duplicateBlobs.heavyJsonPayloadBytes > 16 * 1_048_576) { - throw new Error('Production code graph heavy JSON payload exceeds the bounded surrogate limit.'); + throw new ScriptError('Production code graph heavy JSON payload exceeds the bounded surrogate limit.'); } if (profile.duplicateBlobs.heavyJsonPayloadBytes < profile.lowSignalJsonExclusionThresholdBytes) { - throw new Error('Production code graph heavy JSON payload must reach its declared exclusion threshold.'); + throw new ScriptError('Production code graph heavy JSON payload must reach its declared exclusion threshold.'); } if (profile.lowSignalJsonExclusionThresholdBytes >= profile.highSignalConfigHardCapBytes) { - throw new Error('Production code graph low-signal threshold must remain below the high-signal config hard cap.'); + throw new ScriptError( + 'Production code graph low-signal threshold must remain below the high-signal config hard cap.', + ); } if ( profile.lowSignalJsonExclusionThresholdBytes !== CODE_GRAPH_GENERIC_JSON_EXCLUSION_BYTES || profile.highSignalConfigHardCapBytes !== CODE_GRAPH_HIGH_SIGNAL_JSON_HARD_CAP_BYTES ) { - throw new Error('Production code graph eligibility targets must match the runtime inventory admission policy.'); + throw new ScriptError( + 'Production code graph eligibility targets must match the runtime inventory admission policy.', + ); } productionWorkspaceRoots(profile.workspaceCount, profile.activeWorkspaceExcludedPackageCount); return profile; diff --git a/scripts/code-graph-heavy-tail-fixture.ts b/scripts/code-graph-heavy-tail-fixture.ts index e2cee5a0..dddc9624 100644 --- a/scripts/code-graph-heavy-tail-fixture.ts +++ b/scripts/code-graph-heavy-tail-fixture.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {Effect, FileSystem, Path} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; @@ -54,10 +55,10 @@ export const CODE_GRAPH_HEAVY_TAIL_SMOKE_PROFILE = { } as const satisfies CodeGraphHeavyTailProfile; export function parseCodeGraphHeavyTailProfile(value: unknown): CodeGraphHeavyTailProfile { - if (typeof value !== 'object' || value === null) throw new Error('Heavy-tail profile must be an object.'); + if (typeof value !== 'object' || value === null) throw new ScriptError('Heavy-tail profile must be an object.'); const profile = value as Partial; if (profile.id !== 'large-monorepo-heavy-tail' || profile.version !== 1) { - throw new Error('Unsupported code graph heavy-tail profile.'); + throw new ScriptError('Unsupported code graph heavy-tail profile.'); } for (const field of [ 'callsPerPathologicalTypeScriptFile', @@ -71,16 +72,16 @@ export function parseCodeGraphHeavyTailProfile(value: unknown): CodeGraphHeavyTa ] as const) { const current = profile[field]; if (typeof current !== 'number' || !Number.isSafeInteger(current) || current < 1) { - throw new Error(`Heavy-tail profile ${field} must be a positive safe integer.`); + throw new ScriptError(`Heavy-tail profile ${field} must be a positive safe integer.`); } } - if (profile.parallelWorkers! > 8) throw new Error('Heavy-tail profile parallelWorkers must not exceed 8.'); + if (profile.parallelWorkers! > 8) throw new ScriptError('Heavy-tail profile parallelWorkers must not exceed 8.'); const eligibleFiles = codeGraphHeavyTailEligibleFiles(profile as CodeGraphHeavyTailProfile); if (profile.interruptAfterPersistedFiles! >= eligibleFiles) { - throw new Error('Heavy-tail interruption point must be smaller than the eligible fixture file count.'); + throw new ScriptError('Heavy-tail interruption point must be smaller than the eligible fixture file count.'); } if (profile.lowSignalJsonBytes! < 128 || profile.generatedTypeScriptBytes! < 128) { - throw new Error('Heavy-tail large-file shapes must be at least 128 bytes.'); + throw new ScriptError('Heavy-tail large-file shapes must be at least 128 bytes.'); } return profile as CodeGraphHeavyTailProfile; } @@ -144,7 +145,7 @@ export function codeGraphHeavyTailGeneratedTypeScript(targetBytes: number): stri '', ].join('\n'); if (targetBytes < prefix.length + suffix.length) { - throw new Error(`Generated TypeScript target must be at least ${prefix.length + suffix.length} bytes.`); + throw new ScriptError(`Generated TypeScript target must be at least ${prefix.length + suffix.length} bytes.`); } return `${prefix}${'x'.repeat(targetBytes - prefix.length - suffix.length)}${suffix}`; } @@ -153,7 +154,7 @@ export function codeGraphHeavyTailLowSignalJson(targetBytes: number): string { const prefix = '{"kind":"test-snapshot","frames":[],"payload":"'; const suffix = '"}\n'; if (targetBytes < prefix.length + suffix.length) { - throw new Error(`Low-signal JSON target must be at least ${prefix.length + suffix.length} bytes.`); + throw new ScriptError(`Low-signal JSON target must be at least ${prefix.length + suffix.length} bytes.`); } return `${prefix}${'x'.repeat(targetBytes - prefix.length - suffix.length)}${suffix}`; } diff --git a/scripts/compile-targets.ts b/scripts/compile-targets.ts index 2dae9fa2..6a2ee580 100644 --- a/scripts/compile-targets.ts +++ b/scripts/compile-targets.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Path} from 'effect'; @@ -17,12 +18,12 @@ const compileTargets = Effect.gen(function* () { Effect.flatMap(content => Effect.try({ try: () => JSON.parse(content) as PackageManifest, - catch: cause => new Error('Could not parse package.json.', {cause}), + catch: cause => new ScriptError('Could not parse package.json.', {cause}), }), ), ); if (!manifest.version) { - return yield* Effect.fail(new Error('package.json must declare a version.')); + return yield* Effect.fail(new ScriptError('package.json must declare a version.')); } const configuredTarget = Bun.env.THREADNOTE_BUILD_TARGET?.trim(); @@ -30,7 +31,7 @@ const compileTargets = Effect.gen(function* () { ? BUN_STANDALONE_TARGETS.filter(target => target === configuredTarget) : BUN_STANDALONE_TARGETS; if (targets.length === 0) { - return yield* Effect.fail(new Error(`${configuredTarget} is not a supported standalone target.`)); + return yield* Effect.fail(new ScriptError(`${configuredTarget} is not a supported standalone target.`)); } const outputRoot = path.join(root, '.target-builds'); @@ -63,11 +64,11 @@ const compileTargets = Effect.gen(function* () { sourcemap: 'linked', target: 'bun', }), - catch: cause => new Error(`Bun could not compile ${target}.`, {cause}), + catch: cause => new ScriptError(`Bun could not compile ${target}.`, {cause}), }); if (!result.success) { return yield* Effect.fail( - new Error( + new ScriptError( `${target}: ${result.logs .map(log => log.message) .filter(Boolean) @@ -79,4 +80,4 @@ const compileTargets = Effect.gen(function* () { } }); -BunRuntime.runMain(compileTargets.pipe(Effect.provide(BunServices.layer))); +BunRuntime.runMain(provideScriptLayer(compileTargets, BunServices.layer)); diff --git a/scripts/development-runtime.ts b/scripts/development-runtime.ts index dc89b3fc..48192694 100644 --- a/scripts/development-runtime.ts +++ b/scripts/development-runtime.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {Effect, FileSystem, Option, Path} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; import {sha256FileHex, sha256Hex} from '../src/effect/digest.js'; @@ -77,15 +78,15 @@ interface ReleaseMetadata { export function developmentBuildVersion(packageVersion: string, sourceCommit: string): string { if (!RELEASE_VERSION_PATTERN.test(packageVersion) || !SOURCE_COMMIT_PATTERN.test(sourceCommit)) { - throw new Error('A local development build requires a valid package version and exact Git commit.'); + throw new ScriptError('A local development build requires a valid package version and exact Git commit.'); } if (isDevelopmentBuildVersion(packageVersion)) { - throw new Error('The checked-in package version must not already be a local development version.'); + throw new ScriptError('The checked-in package version must not already be a local development version.'); } const separator = packageVersion.includes('-') ? '.' : '-'; const version = `${packageVersion}${separator}local.g${sourceCommit}`; if (!isDevelopmentBuildVersion(version)) { - throw new Error('Could not derive a valid local development version.'); + throw new ScriptError('Could not derive a valid local development version.'); } return version; } @@ -186,11 +187,15 @@ export const prepareCanonicalDevelopmentInstallRoots = Effect.fn('developmentRun const logicalVersionsRoot = path.join(logicalInstallRoot, 'versions'); yield* fs.makeDirectory(logicalVersionsRoot, {recursive: true, mode: 0o700}); if (Option.isSome(yield* fs.readLink(logicalVersionsRoot).pipe(Effect.option))) { - return yield* Effect.fail(new Error('The managed Threadnote versions directory must not be a symbolic link.')); + return yield* Effect.fail( + new ScriptError('The managed Threadnote versions directory must not be a symbolic link.'), + ); } const realVersionsRoot = yield* fs.realPath(logicalVersionsRoot); if (!canonicalPathEquals(path, system, realVersionsRoot, path.join(realInstallRoot, 'versions'))) { - return yield* Effect.fail(new Error('The managed Threadnote versions directory escapes the installation root.')); + return yield* Effect.fail( + new ScriptError('The managed Threadnote versions directory escapes the installation root.'), + ); } return {installRoot: realInstallRoot, versionsRoot: realVersionsRoot} satisfies CanonicalDevelopmentInstallRoots; }, @@ -212,35 +217,35 @@ export const collectDevelopmentPayloadManifest = Effect.fn('developmentRuntime.c const relative = relativeDirectory ? `${relativeDirectory}/${name}` : name; if (relative === DEVELOPMENT_INSTALL_RECEIPT) continue; if (!isPayloadPath(relative)) { - return yield* Effect.fail(new Error('The development payload contains an invalid relative path.')); + return yield* Effect.fail(new ScriptError('The development payload contains an invalid relative path.')); } if (Option.isSome(yield* fs.readLink(entry).pipe(Effect.option))) { - return yield* Effect.fail(new Error('The development payload must not contain symbolic links.')); + return yield* Effect.fail(new ScriptError('The development payload must not contain symbolic links.')); } const info = yield* fs.stat(entry); const expectedRealPath = path.join(realRoot, ...relative.split('/')); const realEntry = yield* fs.realPath(entry); if (!canonicalPathEquals(path, system, realEntry, expectedRealPath)) { - return yield* Effect.fail(new Error('The development payload contains a non-canonical path.')); + return yield* Effect.fail(new ScriptError('The development payload contains a non-canonical path.')); } if (info.type === 'Directory') { pending.push({directory: entry, relativeDirectory: relative}); continue; } if (info.type !== 'File') { - return yield* Effect.fail(new Error('The development payload contains an unsupported filesystem entry.')); + return yield* Effect.fail(new ScriptError('The development payload contains an unsupported filesystem entry.')); } const linkCount = Option.getOrUndefined(info.nlink); if (linkCount !== undefined && linkCount > 1) { - return yield* Effect.fail(new Error('The development payload must not contain hard-linked files.')); + return yield* Effect.fail(new ScriptError('The development payload must not contain hard-linked files.')); } const size = Number(info.size); if (!Number.isSafeInteger(size) || size < 0) { - return yield* Effect.fail(new Error('The development payload contains a file with an invalid size.')); + return yield* Effect.fail(new ScriptError('The development payload contains a file with an invalid size.')); } if (system.platform !== 'win32' && (info.mode & 0o7000) !== 0) { return yield* Effect.fail( - new Error('The development payload contains a file with unsupported special permission bits.'), + new ScriptError('The development payload contains a file with unsupported special permission bits.'), ); } entries.push({ @@ -265,14 +270,16 @@ export const readManagedDevelopmentRuntimeEvidence = Effect.fn('developmentRunti const path = yield* Path.Path; const system = yield* SystemInfo; if (!SOURCE_COMMIT_PATTERN.test(expectedSourceCommit)) { - return yield* Effect.fail(new Error('Managed development runtime validation requires an exact source commit.')); + return yield* Effect.fail( + new ScriptError('Managed development runtime validation requires an exact source commit.'), + ); } const installRoot = installationRoot(path, system); const active = yield* readJsonOption(fs, path.join(installRoot, 'active-release.json')).pipe( Effect.map(Option.flatMap(parseActiveReleasePointer)), ); if (Option.isNone(active)) { - return yield* Effect.fail(new Error('The managed Threadnote active release pointer is missing or invalid.')); + return yield* Effect.fail(new ScriptError('The managed Threadnote active release pointer is missing or invalid.')); } const logicalVersionsRoot = path.resolve(path.join(installRoot, 'versions')); const [realInstallRoot, realVersionsRoot, realReleaseRoot] = yield* Effect.all([ @@ -281,15 +288,19 @@ export const readManagedDevelopmentRuntimeEvidence = Effect.fn('developmentRunti fs.realPath(active.value.releaseRoot), ]); if (!canonicalPathEquals(path, system, realVersionsRoot, path.join(realInstallRoot, 'versions'))) { - return yield* Effect.fail(new Error('The managed Threadnote versions directory is not canonical.')); + return yield* Effect.fail(new ScriptError('The managed Threadnote versions directory is not canonical.')); } const expectedReleaseRoot = path.join(realVersionsRoot, active.value.version); if (!canonicalPathEquals(path, system, realReleaseRoot, expectedReleaseRoot)) { - return yield* Effect.fail(new Error('The managed Threadnote active release pointer escapes the versions root.')); + return yield* Effect.fail( + new ScriptError('The managed Threadnote active release pointer escapes the versions root.'), + ); } const evidence = yield* readDevelopmentReleaseEvidence(realReleaseRoot, expectedSourceCommit); if (evidence.version !== active.value.version) { - return yield* Effect.fail(new Error('The managed Threadnote active pointer and release version do not match.')); + return yield* Effect.fail( + new ScriptError('The managed Threadnote active pointer and release version do not match.'), + ); } return evidence; }); @@ -306,23 +317,23 @@ export const verifyManagedDevelopmentRuntimeForSource = Effect.fn('developmentRu const live = yield* readStandaloneProcessLeaseVerification(); if (live.truncated) { return yield* Effect.fail( - new Error('Managed development runtime verification could not inspect every live process lease.'), + new ScriptError('Managed development runtime verification could not inspect every live process lease.'), ); } if (live.unverified.length > 0) { return yield* Effect.fail( - new Error('Managed development runtime verification found live process leases with unverified identity.'), + new ScriptError('Managed development runtime verification found live process leases with unverified identity.'), ); } if (live.verified.some(lease => lease.version !== evidence.version)) { return yield* Effect.fail( - new Error('Managed development runtime verification found a process pinned to a superseded release.'), + new ScriptError('Managed development runtime verification found a process pinned to a superseded release.'), ); } const revalidated = yield* readManagedDevelopmentRuntimeEvidence(expectedSourceCommit); if (JSON.stringify(revalidated) !== JSON.stringify(evidence)) { return yield* Effect.fail( - new Error('Managed development runtime verification observed an active release change during preflight.'), + new ScriptError('Managed development runtime verification observed an active release change during preflight.'), ); } return revalidated; @@ -344,7 +355,7 @@ export const verifyManagedDevelopmentRuntimeForSourceCheckout = Effect.fn('devel runtime.sourcePackageManifestSha256 !== sourcePackageManifestSha256 ) { return yield* Effect.fail( - new Error('Managed development runtime dependency evidence does not match the clean source checkout.'), + new ScriptError('Managed development runtime dependency evidence does not match the clean source checkout.'), ); } return runtime; @@ -363,7 +374,7 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read const receipt = Option.flatMap(receiptValue, parseDevelopmentInstallReceipt); const metadata = Option.flatMap(metadataValue, parseReleaseMetadata); if (Option.isNone(receipt) || Option.isNone(metadata)) { - return yield* Effect.fail(new Error('The managed development release metadata or provenance is invalid.')); + return yield* Effect.fail(new ScriptError('The managed development release metadata or provenance is invalid.')); } const sourceFromVersion = developmentVersionSourceCommit(receipt.value.version); if ( @@ -374,11 +385,13 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read metadata.value.target !== receipt.value.target || !releaseTargetMatchesHost(metadata.value.target, system) ) { - return yield* Effect.fail(new Error('The managed development release does not match the exact source commit.')); + return yield* Effect.fail( + new ScriptError('The managed development release does not match the exact source commit.'), + ); } const executableName = system.platform === 'win32' ? 'threadnote.exe' : 'threadnote'; if (metadata.value.executable !== executableName) { - return yield* Effect.fail(new Error('The managed development release executable metadata is invalid.')); + return yield* Effect.fail(new ScriptError('The managed development release executable metadata is invalid.')); } const executable = path.join(releaseRoot, executableName); const [realReleaseParent, realReleaseRoot] = yield* Effect.all([ @@ -386,7 +399,7 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read fs.realPath(releaseRoot), ]); if (!canonicalPathEquals(path, system, realReleaseRoot, path.join(realReleaseParent, path.basename(releaseRoot)))) { - return yield* Effect.fail(new Error('The managed development release directory is not canonical.')); + return yield* Effect.fail(new ScriptError('The managed development release directory is not canonical.')); } const receiptPath = path.join(releaseRoot, DEVELOPMENT_INSTALL_RECEIPT); if ( @@ -398,11 +411,15 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read path.join(realReleaseRoot, DEVELOPMENT_INSTALL_RECEIPT), ) ) { - return yield* Effect.fail(new Error('The managed development release contains a non-canonical provenance file.')); + return yield* Effect.fail( + new ScriptError('The managed development release contains a non-canonical provenance file.'), + ); } const receiptInfo = yield* fs.stat(receiptPath); if (receiptInfo.type !== 'File' || (system.platform !== 'win32' && (receiptInfo.mode & 0o7777) !== 0o600)) { - return yield* Effect.fail(new Error('The managed development release provenance file has unsafe permissions.')); + return yield* Effect.fail( + new ScriptError('The managed development release provenance file has unsafe permissions.'), + ); } const actualPayloadManifest = yield* collectDevelopmentPayloadManifest(releaseRoot); const [actualPayloadManifestSha256, receiptPayloadManifestSha256] = yield* Effect.all([ @@ -414,12 +431,14 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read receiptPayloadManifestSha256 !== receipt.value.payloadManifestSha256 || JSON.stringify(actualPayloadManifest) !== JSON.stringify(receipt.value.payloadManifest) ) { - return yield* Effect.fail(new Error('The managed development release payload manifest does not match its files.')); + return yield* Effect.fail( + new ScriptError('The managed development release payload manifest does not match its files.'), + ); } const executableEntry = actualPayloadManifest.find(entry => entry.path === executableName); const releaseMetadataEntry = actualPayloadManifest.find(entry => entry.path === 'release.json'); if (!executableEntry || !releaseMetadataEntry) { - return yield* Effect.fail(new Error('The managed development release payload omits required files.')); + return yield* Effect.fail(new ScriptError('The managed development release payload omits required files.')); } const [executableInfo, versionResult] = yield* Effect.all( [ @@ -438,7 +457,7 @@ export const readDevelopmentReleaseEvidence = Effect.fn('developmentRuntime.read (system.platform !== 'win32' && (executableInfo.mode & 0o111) === 0) || versionResult.stdout.trim() !== `threadnote v${receipt.value.version}` ) { - return yield* Effect.fail(new Error('The managed development release failed its digest or version check.')); + return yield* Effect.fail(new ScriptError('The managed development release failed its digest or version check.')); } return { dependencyInstallation: receipt.value.dependencyInstallation, @@ -481,7 +500,9 @@ export const stageAndValidateDevelopmentRelease = Effect.fn('developmentRuntime. !canonicalPathEquals(path, system, realVersionsRoot, realStagedParent) || (yield* fs.exists(input.stagedRoot)) ) { - return yield* Effect.fail(new Error('The development staging path is not a fresh child of the versions root.')); + return yield* Effect.fail( + new ScriptError('The development staging path is not a fresh child of the versions root.'), + ); } const distributionManifest = yield* collectDevelopmentPayloadManifest(input.distributionRoot); const distributionManifestSha256 = yield* developmentPayloadManifestSha256(distributionManifest); @@ -489,7 +510,7 @@ export const stageAndValidateDevelopmentRelease = Effect.fn('developmentRuntime. distributionManifestSha256 !== input.receipt.payloadManifestSha256 || JSON.stringify(distributionManifest) !== JSON.stringify(input.receipt.payloadManifest) ) { - return yield* Effect.fail(new Error('The development distribution changed before staging.')); + return yield* Effect.fail(new ScriptError('The development distribution changed before staging.')); } yield* fs.copy(input.distributionRoot, input.stagedRoot, {overwrite: true}); yield* fs.writeFileString( @@ -502,7 +523,7 @@ export const stageAndValidateDevelopmentRelease = Effect.fn('developmentRuntime. } yield* readDevelopmentReleaseEvidence(input.stagedRoot, input.expectedSourceCommit).pipe( Effect.mapError( - cause => new Error('The staged development release failed validation before activation.', {cause}), + cause => new ScriptError('The staged development release failed validation before activation.', {cause}), ), ); return input.stagedRoot; diff --git a/scripts/effect/errors.ts b/scripts/effect/errors.ts new file mode 100644 index 00000000..689a3948 --- /dev/null +++ b/scripts/effect/errors.ts @@ -0,0 +1,19 @@ +import {Effect, Layer} from 'effect'; + +/** Typed failure used at executable-script Effect boundaries. */ +export class ScriptError extends Error { + readonly _tag = 'ScriptError' as const; +} + +export function scriptError(cause: unknown, fallback = 'Threadnote script operation failed.'): ScriptError { + if (cause instanceof ScriptError) return cause; + return new ScriptError(cause instanceof Error ? cause.message : fallback, {cause}); +} + +/** Build a script's terminal service graph with one scoped lifetime boundary. */ +export function provideScriptLayer( + effect: Effect.Effect, + layer: Layer.Layer, +) { + return Effect.scoped(Layer.build(layer).pipe(Effect.flatMap(context => effect.pipe(Effect.provide(context))))); +} diff --git a/scripts/effect/javascript.ts b/scripts/effect/javascript.ts index 7e1bbbb7..72f4192e 100644 --- a/scripts/effect/javascript.ts +++ b/scripts/effect/javascript.ts @@ -14,3 +14,7 @@ export function javascriptStringLiteral(value: string): string { character => JAVASCRIPT_CHARACTER_ESCAPE[character as UnsafeJavascriptStringCharacter], ); } + +export function optionalNativePackageFallbackModule(): string { + return "export const getBinsDir = () => { throw new Error('Optional native package is not included in this Threadnote artifact.'); };"; +} diff --git a/scripts/effect/script.ts b/scripts/effect/script.ts index 2f180138..ae0ede2f 100644 --- a/scripts/effect/script.ts +++ b/scripts/effect/script.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './errors.js'; import {Console, Effect, FileSystem, Path} from 'effect'; import {sha256Hex} from '../../src/effect/digest.js'; import {SystemInfo} from '../../src/effect/system.js'; @@ -17,7 +18,7 @@ export const readJsonFile = Effect.fn('script.readJsonFile')(function* (file: st const raw = yield* fs.readFileString(file); return yield* Effect.try({ try: () => JSON.parse(raw) as unknown, - catch: cause => new Error(`Could not parse JSON file ${file}.`, {cause}), + catch: cause => new ScriptError(`Could not parse JSON file ${file}.`, {cause}), }); }); diff --git a/scripts/evaluate-code-graph-workset.ts b/scripts/evaluate-code-graph-workset.ts index 9d57b67f..67d3c18b 100644 --- a/scripts/evaluate-code-graph-workset.ts +++ b/scripts/evaluate-code-graph-workset.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; @@ -60,7 +61,7 @@ export function parseCodeGraphWorksetEvaluationArguments( else if (argument === '--output') outputPath = required(args[++index], argument); else if (argument === '--sizes') requestedSizes = parseEvaluationSizes(required(args[++index], argument)); else if (argument === '--smoke') smoke = true; - else throw new Error(`Unknown code graph workset evaluation option: ${argument}`); + else throw new ScriptError(`Unknown code graph workset evaluation option: ${argument}`); } const sizes = requestedSizes ?? (smoke ? WORKSET_EVALUATION_SMOKE_SIZES : CODE_GRAPH_WORKSET_FIXTURE_SIZES); return {createdAt, outputPath, sizes, smoke}; @@ -74,7 +75,7 @@ export const evaluateCodeGraphWorkset = Effect.scoped( yield* indexPreparedCodeGraphWorksetFixture(prepared); const selectedWorksets = options.sizes.map(size => { const workset = prepared.plan.worksets.find(candidate => candidate.size === size); - if (!workset) throw new Error(`Fixture did not emit a size-${size} workset.`); + if (!workset) throw new ScriptError(`Fixture did not emit a size-${size} workset.`); return workset.name; }); yield* publishIndexedCodeGraphWorksetCatalog(prepared, selectedWorksets); @@ -84,7 +85,7 @@ export const evaluateCodeGraphWorkset = Effect.scoped( const observations: CodeGraphWorksetEvaluationObservationV1[] = []; for (const worksetSize of options.sizes) { const workset = prepared.plan.worksets.find(candidate => candidate.size === worksetSize); - if (!workset) return yield* Effect.fail(new Error(`Fixture did not emit a size-${worksetSize} workset.`)); + if (!workset) return yield* Effect.fail(new ScriptError(`Fixture did not emit a size-${worksetSize} workset.`)); const queries = fixture.queries.filter(query => query.sizes.includes(worksetSize)); const worktree = yield* measureWorktreeIsolation(prepared, config, workset.name, worksetSize); let coverage: readonly CodeGraphWorksetCoverageObservationV1[] | undefined; @@ -96,7 +97,9 @@ export const evaluateCodeGraphWorkset = Effect.scoped( candidate => candidate.operation === 'query' && candidate.sizes.includes(worksetSize), ); if (!probe?.query) { - return yield* Effect.fail(new Error(`Fixture size ${worksetSize} has no executable coverage probe.`)); + return yield* Effect.fail( + new ScriptError(`Fixture size ${worksetSize} has no executable coverage probe.`), + ); } const measured = yield* measureCodeGraphWorksetQuery(config, workset.name, probe.query); coverage = codeGraphWorksetCoverage(fixture, worksetSize, measured.result); @@ -124,7 +127,7 @@ export const evaluateCodeGraphWorkset = Effect.scoped( const metrics = evaluateCodeGraphWorksetObservations(fixture, observations); const system = yield* SystemInfo; - const hardware = yield* system.hardwareInfo(); + const hardware = yield* system.hardwareInfo; const [commit, dirty, version] = yield* Effect.all( [ sourceGit(['rev-parse', 'HEAD']), @@ -156,7 +159,7 @@ export const evaluateCodeGraphWorkset = Effect.scoped( if (options.outputPath) yield* atomicWrite(options.outputPath, `${JSON.stringify(baseline, undefined, 2)}\n`); yield* printJson(baseline); const safetyFailures = codeGraphWorksetEvaluationSafetyFailures(metrics); - if (safetyFailures.length > 0) return yield* Effect.fail(new Error(safetyFailures.join('\n'))); + if (safetyFailures.length > 0) return yield* Effect.fail(new ScriptError(safetyFailures.join('\n'))); }), ); @@ -178,12 +181,12 @@ function acquirePreparedFixture(size: CodeGraphWorksetFixtureSize, stateProfile: return Effect.acquireRelease( Effect.tryPromise({ try: () => prepareCodeGraphWorksetFixture({size, stateProfile}), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }), fixture => Effect.tryPromise({ try: () => removePreparedCodeGraphWorksetFixture(fixture), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }).pipe(Effect.catch(() => Effect.void)), ); } @@ -211,7 +214,7 @@ function parseEvaluationSizes(value: string): readonly CodeGraphWorksetFixtureSi const sizes = parseSizeList(value, '--sizes'); for (const size of sizes) { if (!allowed.has(size)) { - throw new Error( + throw new ScriptError( `--sizes only accepts evaluation sizes: ${CODE_GRAPH_WORKSET_FIXTURE_SIZES.join(', ')}. Received ${size}.`, ); } @@ -222,32 +225,33 @@ function parseEvaluationSizes(value: string): readonly CodeGraphWorksetFixtureSi function parseSizeList(value: string, option: string): readonly number[] { const parts = value.split(','); if (parts.length === 0 || parts.some(part => !part.trim())) - throw new Error(`${option} requires comma-separated sizes.`); + throw new ScriptError(`${option} requires comma-separated sizes.`); const sizes = parts.map(part => Number(part.trim())); if (sizes.some(size => !Number.isSafeInteger(size) || size < 1)) { - throw new Error(`${option} requires positive integer sizes.`); + throw new ScriptError(`${option} requires positive integer sizes.`); } - if (new Set(sizes).size !== sizes.length) throw new Error(`${option} sizes must be unique.`); + if (new Set(sizes).size !== sizes.length) throw new ScriptError(`${option} sizes must be unique.`); return sizes; } function defaultCreatedAt(environment: NodeJS.ProcessEnv, now: Date): string { const epoch = environment.SOURCE_DATE_EPOCH; if (epoch === undefined) return parseCreatedAt(now.toISOString(), 'current time'); - if (!/^\d+$/.test(epoch)) throw new Error('SOURCE_DATE_EPOCH must be a non-negative integer number of seconds.'); + if (!/^\d+$/.test(epoch)) + throw new ScriptError('SOURCE_DATE_EPOCH must be a non-negative integer number of seconds.'); const date = new Date(Number(epoch) * 1_000); - if (!Number.isFinite(date.getTime())) throw new Error('SOURCE_DATE_EPOCH is outside the supported date range.'); + if (!Number.isFinite(date.getTime())) throw new ScriptError('SOURCE_DATE_EPOCH is outside the supported date range.'); return date.toISOString(); } function parseCreatedAt(value: string, option: string): string { const date = new Date(value); - if (!Number.isFinite(date.getTime())) throw new Error(`${option} requires a valid date.`); + if (!Number.isFinite(date.getTime())) throw new ScriptError(`${option} requires a valid date.`); return date.toISOString(); } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } @@ -258,4 +262,4 @@ const sourceGit = Effect.fn('evaluateCodeGraphWorkset.sourceGit')((args: readonl }).pipe(Effect.map(result => result.stdout.trim())), ); -if (import.meta.main) BunRuntime.runMain(evaluateCodeGraphWorkset.pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(evaluateCodeGraphWorkset, ApplicationLayer)); diff --git a/scripts/evaluate-code-graph.ts b/scripts/evaluate-code-graph.ts index 3fc0bcf3..f480b189 100644 --- a/scripts/evaluate-code-graph.ts +++ b/scripts/evaluate-code-graph.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect, FileSystem, Path} from 'effect'; import {codeGraphLayout} from '../src/code_graph/layout.js'; @@ -140,7 +141,7 @@ const evaluateNativeCodeGraph = Effect.scoped( metrics, observations, }); - return yield* Effect.fail(new Error(gateFailures.join('\n'))); + return yield* Effect.fail(new ScriptError(gateFailures.join('\n'))); } const baseline: CodeGraphEvaluationBaselineV1 = { createdAt: options.createdAt, @@ -206,14 +207,14 @@ function parseArguments(args: readonly string[]): { if (argument === '--output') outputPath = required(args[++index], argument); else if (argument === '--fixture') fixture = required(args[++index], argument); else if (argument === '--created-at') createdAt = new Date(required(args[++index], argument)).toISOString(); - else throw new Error(`Unknown code graph evaluation option: ${argument}`); + else throw new ScriptError(`Unknown code graph evaluation option: ${argument}`); } - if (!/^code-graph-[a-z0-9-]+$/.test(fixture)) throw new Error(`Invalid code graph fixture name: ${fixture}.`); + if (!/^code-graph-[a-z0-9-]+$/.test(fixture)) throw new ScriptError(`Invalid code graph fixture name: ${fixture}.`); return {createdAt, fixture, outputPath}; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } @@ -224,8 +225,9 @@ const replaceContractSymbol = Effect.fn('codeGraphEvaluation.replaceContractSymb to: string, ) { const content = yield* fs.readFileString(target); - if (!content.includes(from)) return yield* Effect.fail(new Error(`Evaluation fixture does not contain ${from}.`)); + if (!content.includes(from)) + return yield* Effect.fail(new ScriptError(`Evaluation fixture does not contain ${from}.`)); yield* fs.writeFileString(target, content.replaceAll(from, to)); }); -BunRuntime.runMain(evaluateNativeCodeGraph.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(evaluateNativeCodeGraph, ApplicationLayer)); diff --git a/scripts/evaluate-recall-models.ts b/scripts/evaluate-recall-models.ts index 41fdd3b6..7aaa0ecd 100644 --- a/scripts/evaluate-recall-models.ts +++ b/scripts/evaluate-recall-models.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect, FileSystem, Path} from 'effect'; import {LocalModelRuntime} from '../src/effect/ai/local-model-runtime.js'; @@ -30,7 +31,9 @@ const evaluateModels = Effect.gen(function* () { const baseline = parseRecallEvaluationBaselineV1(yield* readJsonFile(options.baseline)); if (baseline.fixture.hash !== hash) { return yield* Effect.fail( - new Error(`Recall baseline fixture hash ${baseline.fixture.hash} does not match generated fixture hash ${hash}.`), + new ScriptError( + `Recall baseline fixture hash ${baseline.fixture.hash} does not match generated fixture hash ${hash}.`, + ), ); } const embeddingManifest = options.embedding ? builtinManifest(options.embedding, 'embedding') : undefined; @@ -38,7 +41,7 @@ const evaluateModels = Effect.gen(function* () { ? parseLocalModelManifest(yield* readJsonFile(options.rerankerManifest)) : undefined; if (localRerankerManifest && localRerankerManifest.role !== 'reranker') { - return yield* Effect.fail(new Error(`Local model ${localRerankerManifest.id} is not a reranker.`)); + return yield* Effect.fail(new ScriptError(`Local model ${localRerankerManifest.id} is not a reranker.`)); } const rerankerManifest = localRerankerManifest ?? (options.reranker ? builtinManifest(options.reranker, 'reranker') : undefined); @@ -47,7 +50,7 @@ const evaluateModels = Effect.gen(function* () { ); if (manifests.length === 0) { return yield* Effect.fail( - new Error( + new ScriptError( 'Pass --embedding , --reranker , or a local --reranker-manifest/--reranker-path pair.', ), ); @@ -186,7 +189,7 @@ const evaluateModels = Effect.gen(function* () { function builtinManifest(id: string, role: 'embedding' | 'reranker'): LocalModelManifest { const candidate = BUILTIN_MODEL_MANIFESTS.find(value => value.id === id); - if (!candidate || candidate.role !== role) throw new Error(`Unknown ${role} model: ${id}`); + if (!candidate || candidate.role !== role) throw new ScriptError(`Unknown ${role} model: ${id}`); return candidate; } @@ -232,13 +235,13 @@ function parseArguments(args: readonly string[], resolve: (value: string) => str else if (argument === '--reranker-manifest') rerankerManifest = resolve(required(args[++index], argument)); else if (argument === '--reranker-path') rerankerPath = resolve(required(args[++index], argument)); else if (argument === '--summary-output') summaryOutput = required(args[++index], argument); - else throw new Error(`Unknown model-evaluation option: ${argument}`); + else throw new ScriptError(`Unknown model-evaluation option: ${argument}`); } if ((rerankerManifest === undefined) !== (rerankerPath === undefined)) { - throw new Error('--reranker-manifest and --reranker-path must be passed together.'); + throw new ScriptError('--reranker-manifest and --reranker-path must be passed together.'); } if (reranker && rerankerManifest) { - throw new Error('--reranker cannot be combined with --reranker-manifest.'); + throw new ScriptError('--reranker cannot be combined with --reranker-manifest.'); } return { baseline, @@ -260,19 +263,19 @@ const verifyLocalModelArtifact = Effect.fn('evaluateRecallModels.verifyLocalMode ) { const fs = yield* FileSystem.FileSystem; const info = yield* fs.stat(modelPath); - if (info.type !== 'File') throw new Error(`Local reranker artifact is not a regular file: ${modelPath}`); + if (info.type !== 'File') throw new ScriptError(`Local reranker artifact is not a regular file: ${modelPath}`); if (Number(info.size) !== manifest.size) { - throw new Error(`Local reranker size ${info.size} does not match manifest size ${manifest.size}.`); + throw new ScriptError(`Local reranker size ${info.size} does not match manifest size ${manifest.size}.`); } const digest = yield* sha256FileHex(modelPath); if (digest !== manifest.sha256) { - throw new Error(`Local reranker SHA-256 ${digest} does not match manifest SHA-256 ${manifest.sha256}.`); + throw new ScriptError(`Local reranker SHA-256 ${digest} does not match manifest SHA-256 ${manifest.sha256}.`); } }); function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } -BunRuntime.runMain(evaluateModels.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(evaluateModels, ApplicationLayer)); diff --git a/scripts/evaluate-recall-v2.ts b/scripts/evaluate-recall-v2.ts index 8e1d69a6..866282ea 100644 --- a/scripts/evaluate-recall-v2.ts +++ b/scripts/evaluate-recall-v2.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import {Effect} from 'effect'; import {ApplicationLayer} from '../src/effect/runtime.js'; @@ -27,7 +28,9 @@ const evaluateRecall = Effect.gen(function* () { : undefined; if (baseline && baseline.fixture.hash !== hash) { return yield* Effect.fail( - new Error(`Recall baseline fixture hash ${baseline.fixture.hash} does not match generated fixture hash ${hash}`), + new ScriptError( + `Recall baseline fixture hash ${baseline.fixture.hash} does not match generated fixture hash ${hash}`, + ), ); } const gate = baseline ? evaluateRecallNonInferiority(baselineResult(baseline), result) : undefined; @@ -98,10 +101,10 @@ function parseArguments(args: readonly string[]): EvaluationOptions { else if (argument === '--max-failures') maximumPrintedFailures = positiveInteger(args[++index], '--max-failures'); else if (argument === '--output') outputPath = requiredValue(args[++index], '--output'); else if (argument === '--seed') seed = positiveInteger(args[++index], '--seed'); - else throw new Error(`Unknown recall-v2 evaluation option: ${argument}`); + else throw new ScriptError(`Unknown recall-v2 evaluation option: ${argument}`); } if (failOnRegression && !baselinePath) { - throw new Error('--fail-on-regression requires --baseline '); + throw new ScriptError('--fail-on-regression requires --baseline '); } return { baselinePath, @@ -118,14 +121,14 @@ function parseArguments(args: readonly string[]): EvaluationOptions { function positiveInteger(value: string | undefined, option: string): number { const parsed = Number.parseInt(requiredValue(value, option), 10); if (!Number.isSafeInteger(parsed) || parsed < 1) { - throw new Error(`${option} requires a positive integer`); + throw new ScriptError(`${option} requires a positive integer`); } return parsed; } function requiredValue(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value`); return value; } -BunRuntime.runMain(evaluateRecall.pipe(Effect.provide(ApplicationLayer))); +BunRuntime.runMain(provideScriptLayer(evaluateRecall, ApplicationLayer)); diff --git a/scripts/evaluate-recall.ts b/scripts/evaluate-recall.ts index 31a28231..a06c3d0d 100644 --- a/scripts/evaluate-recall.ts +++ b/scripts/evaluate-recall.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import {BunRuntime} from '@effect/platform-bun'; import {Clock, Console, Effect, FileSystem, Option, Path} from 'effect'; import {ApplicationLayer} from '../src/effect/runtime.js'; @@ -33,7 +34,7 @@ const program = Effect.gen(function* () { const raw = yield* fs.readFileString(FIXTURE_PATH); const fixture = yield* Effect.try({ try: () => parseRecallEvaluationFixture(JSON.parse(raw)), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }); const durations: number[] = []; for (const query of fixture.queries) { @@ -55,12 +56,12 @@ const program = Effect.gen(function* () { }; yield* Console.log(JSON.stringify(output, undefined, 2)); if (result.failures.length > 0) { - return yield* Effect.fail(new Error(`Recall evaluation failed ${result.failures.length} contract check(s).`)); + return yield* Effect.fail(new ScriptError(`Recall evaluation failed ${result.failures.length} contract check(s).`)); } for (const [name, benchmark] of Object.entries(productionBenchmark.scenarios)) { if (benchmark.p95Milliseconds > benchmark.p95LimitMilliseconds) { return yield* Effect.fail( - new Error( + new ScriptError( `Recall ${name} benchmark p95 ${benchmark.p95Milliseconds.toFixed(2)}ms exceeds ${benchmark.p95LimitMilliseconds}ms.`, ), ); @@ -118,7 +119,7 @@ const runProductionBenchmark = Effect.scoped( const targetInfo = yield* fs.stat(targetRepoPath); const modifiedAt = Option.getOrUndefined(targetInfo.mtime)?.getTime(); if (modifiedAt === undefined) { - return yield* Effect.fail(new Error('Production benchmark target has no modification time.')); + return yield* Effect.fail(new ScriptError('Production benchmark target has no modification time.')); } yield* fs.writeFileString( seedStatePath, @@ -140,7 +141,9 @@ const runProductionBenchmark = Effect.scoped( initialIndex.find(candidate => candidate.uri === 'threadnote://resources/repos/threadnote/09999.md') ?.authority !== 'canonical_repo' ) { - return yield* Effect.fail(new Error('Production benchmark target did not receive verified seed authority.')); + return yield* Effect.fail( + new ScriptError('Production benchmark target did not receive verified seed authority.'), + ); } const prepareForQuery = (query: string) => prepareRecallSections(config, { @@ -158,7 +161,7 @@ const runProductionBenchmark = Effect.scoped( for (let index = 0; index < PRODUCTION_BENCHMARK_WARMUP_COUNT; index += 1) { const warmup = yield* prepare; if (!warmup.ranked.some(hit => hit.uri === 'threadnote://resources/repos/threadnote/09999.md')) { - return yield* Effect.fail(new Error('Production benchmark failed to retrieve its exact target.')); + return yield* Effect.fail(new ScriptError('Production benchmark failed to retrieve its exact target.')); } } const targetUri = 'threadnote://resources/repos/threadnote/09999.md'; @@ -176,7 +179,7 @@ const runProductionBenchmark = Effect.scoped( const result = yield* runSample(sample); const finishedAt = yield* Clock.currentTimeNanos; if (!result.ranked.some(hit => hit.uri === targetUri)) { - return yield* Effect.fail(new Error('Production benchmark failed to retrieve its target.')); + return yield* Effect.fail(new ScriptError('Production benchmark failed to retrieve its target.')); } durations.push(Number(finishedAt - startedAt) / NANOSECONDS_PER_MILLISECOND); } @@ -203,7 +206,7 @@ const runProductionBenchmark = Effect.scoped( const found = matches.some(match => match.uri === targetUri); if ((expected === 'hit' && !found) || (expected === 'no-hit' && matches.length > 0)) { return yield* Effect.fail( - new Error(`Production exact-search benchmark produced an unexpected ${expected}.`), + new ScriptError(`Production exact-search benchmark produced an unexpected ${expected}.`), ); } durations.push(Number(finishedAt - startedAt) / NANOSECONDS_PER_MILLISECOND); @@ -247,7 +250,7 @@ const runProductionBenchmark = Effect.scoped( const finalUnchangedQuery = yield* prepare; if (!finalUnchangedQuery.ranked.some(hit => hit.uri === targetUri)) { return yield* Effect.fail( - new Error('Production benchmark lost an unchanged-term target after sustained incremental updates.'), + new ScriptError('Production benchmark lost an unchanged-term target after sustained incremental updates.'), ); } return { @@ -272,6 +275,6 @@ function percentile(sorted: readonly number[], quantile: number): number { return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * quantile))] ?? 0; } -BunRuntime.runMain(program.pipe(Effect.provide(ApplicationLayer)), { +BunRuntime.runMain(provideScriptLayer(program, ApplicationLayer), { disableErrorReporting: false, }); diff --git a/scripts/generate-code-graph-language-catalog.ts b/scripts/generate-code-graph-language-catalog.ts index f8b5a554..a3c54d00 100644 --- a/scripts/generate-code-graph-language-catalog.ts +++ b/scripts/generate-code-graph-language-catalog.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Path} from 'effect'; @@ -18,7 +19,7 @@ const generateCatalog = Effect.gen(function* () { if (!(yield* fs.exists(path.join(candidate, 'pack.ts')))) continue; packs.push({alias: safeIdentifier(directory), directory}); } - if (packs.length === 0) return yield* Effect.fail(new Error('No code graph language packs were discovered.')); + if (packs.length === 0) return yield* Effect.fail(new ScriptError('No code graph language packs were discovered.')); const output = [ '// Generated by scripts/generate-code-graph-language-catalog.ts. Do not edit by hand.', ...packs.map(pack => `import {codeGraphLanguagePack as ${pack.alias}} from './${pack.directory}/pack.js';`), @@ -39,4 +40,4 @@ function safeIdentifier(value: string): string { return /^[A-Za-z_$]/.test(identifier) ? identifier : `pack${identifier}`; } -BunRuntime.runMain(generateCatalog.pipe(Effect.provide(BunServices.layer))); +BunRuntime.runMain(provideScriptLayer(generateCatalog, BunServices.layer)); diff --git a/scripts/generate-mixed-nx-bazel-fixture.ts b/scripts/generate-mixed-nx-bazel-fixture.ts index 0a920600..bbd55f3e 100644 --- a/scripts/generate-mixed-nx-bazel-fixture.ts +++ b/scripts/generate-mixed-nx-bazel-fixture.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import {BunRuntime} from '@effect/platform-bun'; import {Effect, FileSystem, Path} from 'effect'; import {runCommandEffect} from '../src/effect/command.js'; @@ -34,9 +35,9 @@ export function parseMixedNxBazelFixtureArguments(args: readonly string[]): Mixe for (let index = 0; index < args.length; index += 1) { const argument = args[index]; if (argument === '--output') output = args[++index]; - else throw new Error(`Unknown mixed-monorepo fixture option: ${argument}`); + else throw new ScriptError(`Unknown mixed-monorepo fixture option: ${argument}`); } - if (!output?.trim()) throw new Error('--output requires a path.'); + if (!output?.trim()) throw new ScriptError('--output requires a path.'); return {output}; } @@ -47,7 +48,7 @@ export const generateMixedNxBazelFixture = Effect.fn('mixedNxBazelFixture.genera const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const output = path.resolve(options.output); - if (yield* fs.exists(output)) throw new Error(`Fixture output already exists: ${output}`); + if (yield* fs.exists(output)) throw new ScriptError(`Fixture output already exists: ${output}`); const [nx, rulesJs, angular] = MIXED_NX_BAZEL_FIXTURE_SOURCES; yield* cloneExact(nx.repository, nx.commit, output); @@ -107,10 +108,10 @@ const cloneExact = Effect.fn('mixedNxBazelFixture.cloneExact')(function* ( if (sparsePaths) yield* runGit(target, ['sparse-checkout', 'set', '--no-cone', ...sparsePaths]); yield* runGit(target, ['checkout', '--quiet', commit], 10 * 60_000); const resolved = (yield* runGit(target, ['rev-parse', 'HEAD'])).stdout.trim(); - if (resolved !== commit) throw new Error(`Fixture source resolved ${resolved} instead of ${commit}.`); + if (resolved !== commit) throw new ScriptError(`Fixture source resolved ${resolved} instead of ${commit}.`); }); const runGit = (cwd: string, args: readonly string[], timeoutMs = 60_000) => runCommandEffect('git', ['-C', cwd, ...args], {maxOutputBytes: 64 * 1_024, timeoutMs}); -if (import.meta.main) BunRuntime.runMain(generateMixedNxBazelFixture().pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(generateMixedNxBazelFixture(), ApplicationLayer)); diff --git a/scripts/install-local-standalone.ts b/scripts/install-local-standalone.ts index 66371716..228f89c7 100644 --- a/scripts/install-local-standalone.ts +++ b/scripts/install-local-standalone.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Cause, Console, Crypto, Effect, Exit, FileSystem, Layer, Option, Path} from 'effect'; @@ -110,7 +111,7 @@ export function parseLocalStandaloneInstallArguments(arguments_: readonly string if (argument === '--json') json = true; else if (argument === '--take-over-global-runtime') takeOverGlobalRuntime = true; else if (argument === '--terminate-superseded') terminateSuperseded = true; - else throw new Error(`Unknown local standalone install option: ${argument}`); + else throw new ScriptError(`Unknown local standalone install option: ${argument}`); } return {json, takeOverGlobalRuntime, terminateSuperseded}; } @@ -135,7 +136,7 @@ export const installLocalStandalone = Effect.fn('developmentInstall.run')(functi const sourceRoot = yield* path.fromFileUrl(ROOT_URL); const git = Option.fromNullishOr(Bun.which('git')); if (Option.isNone(git)) - return yield* Effect.fail(new Error('Git is required for an exact-HEAD development install.')); + return yield* Effect.fail(new ScriptError('Git is required for an exact-HEAD development install.')); const [sourceCommit, status] = yield* Effect.all( [ runCommandEffect(git.value, ['rev-parse', 'HEAD'], {cwd: sourceRoot}), @@ -145,10 +146,12 @@ export const installLocalStandalone = Effect.fn('developmentInstall.run')(functi ); const commit = sourceCommit.stdout.trim(); if (!GIT_COMMIT_PATTERN.test(commit)) { - return yield* Effect.fail(new Error('The Threadnote checkout did not resolve to an exact Git commit.')); + return yield* Effect.fail(new ScriptError('The Threadnote checkout did not resolve to an exact Git commit.')); } if (status.stdout.length > 0) { - return yield* Effect.fail(new Error('Refusing a global development install from a dirty Threadnote checkout.')); + return yield* Effect.fail( + new ScriptError('Refusing a global development install from a dirty Threadnote checkout.'), + ); } const manifest = yield* readPackageManifest(fs, path.join(sourceRoot, 'package.json')); const version = developmentBuildVersion(manifest.version, commit); @@ -218,7 +221,7 @@ const verifyCleanSourceState = Effect.fn('developmentInstall.verifyCleanSourceSt expectedCommit: string, ) { const git = Option.fromNullishOr(Bun.which('git')); - if (Option.isNone(git)) return yield* Effect.fail(new Error('Git disappeared before development activation.')); + if (Option.isNone(git)) return yield* Effect.fail(new ScriptError('Git disappeared before development activation.')); const [commit, status] = yield* Effect.all( [ runCommandEffect(git.value, ['rev-parse', 'HEAD'], {cwd: sourceRoot}), @@ -227,7 +230,7 @@ const verifyCleanSourceState = Effect.fn('developmentInstall.verifyCleanSourceSt {concurrency: 2}, ); if (commit.stdout.trim() !== expectedCommit || status.stdout.length > 0) { - return yield* Effect.fail(new Error('The Threadnote checkout changed before development activation.')); + return yield* Effect.fail(new ScriptError('The Threadnote checkout changed before development activation.')); } }); @@ -246,7 +249,7 @@ const requireDevelopmentRuntimeOwnership = Effect.fn('developmentInstall.require takeOverGlobalRuntime: boolean, ) { if (!SHA256_PATTERN.test(requestedSourceCheckoutId)) { - return yield* Effect.fail(new Error('The development source checkout identity is invalid.')); + return yield* Effect.fail(new ScriptError('The development source checkout identity is invalid.')); } const [activeVersion, owner] = yield* Effect.all([ activeInstalledVersion(), @@ -261,7 +264,7 @@ const requireDevelopmentRuntimeOwnership = Effect.fn('developmentInstall.require ? 'the active global development runtime changed outside its owning installer' : 'the active global development runtime ownership record is invalid'; return yield* Effect.fail( - new Error( + new ScriptError( `Refusing to replace the global Threadnote runtime because ${reason}. ` + 'Rerun with --take-over-global-runtime only after confirming the other development task has finished.', ), @@ -352,14 +355,16 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti if (yield* fs.exists(input.releaseRoot)) { const concurrentEvidence = yield* readDevelopmentReleaseEvidence(input.releaseRoot, input.commit).pipe( Effect.mapError( - cause => new Error('A concurrent exact-version development release is not reusable.', {cause}), + cause => new ScriptError('A concurrent exact-version development release is not reusable.', {cause}), ), ); yield* requireEvidenceVersion(concurrentEvidence, input.version); reused = true; } else { const stagedEvidence = yield* readDevelopmentReleaseEvidence(input.stagedRoot.value, input.commit).pipe( - Effect.mapError(cause => new Error('The staged development release changed before activation.', {cause})), + Effect.mapError( + cause => new ScriptError('The staged development release changed before activation.', {cause}), + ), ); yield* requireEvidenceVersion(stagedEvidence, input.version); yield* promoteStandaloneReleaseDirectory(fs, path, input.stagedRoot.value, input.releaseRoot, system.processId); @@ -370,7 +375,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti const releaseEvidence = yield* readDevelopmentReleaseEvidence(input.releaseRoot, input.commit).pipe( Effect.mapError( cause => - new Error( + new ScriptError( input.reused ? 'The existing exact-version development release is not reusable.' : 'The promoted development release failed validation.', @@ -386,7 +391,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti }); if (!doctor.stdout.includes('Running Threadnote doctor checks.') || !doctor.stdout.includes('Summary:')) { return yield* Effect.fail( - new Error('The installed development executable did not complete doctor verification.'), + new ScriptError('The installed development executable did not complete doctor verification.'), ); } return yield* Effect.all([ @@ -407,7 +412,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti Effect.matchCauseEffect({ onFailure: cleanupCause => Effect.fail( - new Error('The new development release failed validation and could not be removed.', { + new ScriptError('The new development release failed validation and could not be removed.', { cause: new AggregateError([Cause.squash(validationCause), Cause.squash(cleanupCause)]), }), ), @@ -434,7 +439,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti Effect.matchCauseEffect({ onFailure: rollbackCause => Effect.fail( - new Error('Development release activation failed and rollback was incomplete.', { + new ScriptError('Development release activation failed and rollback was incomplete.', { cause: new AggregateError([Cause.squash(activationCause), Cause.squash(rollbackCause)]), }), ), @@ -482,7 +487,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti yield* fs.remove(stagedRoot, {force: true, recursive: true}); } if (yield* fs.exists(stagedRoot)) { - return yield* Effect.fail(new Error('The development staging directory still exists after cleanup.')); + return yield* Effect.fail(new ScriptError('The development staging directory still exists after cleanup.')); } }), ); @@ -514,7 +519,7 @@ export const activateLocalStandaloneRelease = Effect.fn('developmentInstall.acti function requireEvidenceVersion(evidence: DevelopmentRuntimeEvidence, expectedVersion: string) { return evidence.version === expectedVersion ? Effect.void - : Effect.fail(new Error('The validated development release version does not match its activation target.')); + : Effect.fail(new ScriptError('The validated development release version does not match its activation target.')); } interface LocalFileSnapshot { @@ -537,7 +542,9 @@ const captureFileSnapshot = Effect.fn('developmentInstall.captureFileSnapshot')( fs.stat(file).pipe(Effect.option), ]); if (Option.isSome(link) || (exists && Option.isNone(content))) { - return yield* Effect.fail(new Error('A managed installation file cannot be safely snapshotted for rollback.')); + return yield* Effect.fail( + new ScriptError('A managed installation file cannot be safely snapshotted for rollback.'), + ); } return { content, @@ -557,7 +564,7 @@ const restoreFileSnapshots = Effect.fn('developmentInstall.restoreFileSnapshots' for (const snapshot of snapshots) { const restored = yield* Effect.exit(restoreFileSnapshot(fs, path, system, snapshot)); if (Exit.isFailure(restored)) { - failures.push(new Error(`Could not restore the ${snapshot.label}.`, {cause: Cause.squash(restored.cause)})); + failures.push(new ScriptError(`Could not restore the ${snapshot.label}.`, {cause: Cause.squash(restored.cause)})); } } if (failures.length > 0) { @@ -624,7 +631,7 @@ const buildAndStageDevelopmentRelease = Effect.fn('developmentInstall.buildAndSt timeoutMs: COMMAND_TIMEOUT_MILLISECONDS, }); const git = Option.fromNullishOr(Bun.which('git')); - if (Option.isNone(git)) return yield* Effect.fail(new Error('Git disappeared during the development build.')); + if (Option.isNone(git)) return yield* Effect.fail(new ScriptError('Git disappeared during the development build.')); const [afterCommit, afterStatus] = yield* Effect.all( [ runCommandEffect(git.value, ['rev-parse', 'HEAD'], {cwd: input.sourceRoot}), @@ -633,13 +640,15 @@ const buildAndStageDevelopmentRelease = Effect.fn('developmentInstall.buildAndSt {concurrency: 2}, ); if (afterCommit.stdout.trim() !== input.commit || afterStatus.stdout.length > 0) { - return yield* Effect.fail(new Error('The Threadnote checkout changed while building the development executable.')); + return yield* Effect.fail( + new ScriptError('The Threadnote checkout changed while building the development executable.'), + ); } const distributionRoot = path.join(input.sourceRoot, 'dist'); const releaseMetadataPath = path.join(distributionRoot, 'release.json'); const releaseMetadata = yield* readReleaseMetadata(fs, releaseMetadataPath); if (releaseMetadata.version !== input.version || releaseMetadata.executable !== input.executableName) { - return yield* Effect.fail(new Error('The development build did not embed its exact SHA-bound version.')); + return yield* Effect.fail(new ScriptError('The development build did not embed its exact SHA-bound version.')); } const executable = path.join(distributionRoot, input.executableName); const payloadManifest = yield* collectDevelopmentPayloadManifest(distributionRoot); @@ -662,7 +671,7 @@ const buildAndStageDevelopmentRelease = Effect.fn('developmentInstall.buildAndSt {concurrency: 6}, ); if (versionResult.stdout.trim() !== `threadnote v${input.version}`) { - return yield* Effect.fail(new Error('The compiled development executable reported the wrong version.')); + return yield* Effect.fail(new ScriptError('The compiled development executable reported the wrong version.')); } const receipt: DevelopmentInstallReceiptV1 = { builtAt: new Date().toISOString(), @@ -711,14 +720,16 @@ const verifyLaunchers = Effect.fn('developmentInstall.verifyLaunchers')(function const [launcher, expected] = yield* Effect.all([commandLauncherPath(mode), renderCommandShim(releaseRoot, mode)]); const actual = yield* fs.readFileString(launcher); if (actual !== expected) { - return yield* Effect.fail(new Error(`The managed ${mode} launcher did not activate the development release.`)); + return yield* Effect.fail( + new ScriptError(`The managed ${mode} launcher did not activate the development release.`), + ); } if (system.platform !== 'win32') { const info = yield* fs.stat(launcher); if ((info.mode & 0o777) !== 0o755) yield* fs.chmod(launcher, 0o755); const repaired = yield* fs.stat(launcher); if ((repaired.mode & 0o777) !== 0o755) { - return yield* Effect.fail(new Error(`The managed ${mode} launcher does not have safe executable mode.`)); + return yield* Effect.fail(new ScriptError(`The managed ${mode} launcher does not have safe executable mode.`)); } } if (mode === 'cli') cliLauncher = launcher; @@ -729,7 +740,9 @@ const verifyLaunchers = Effect.fn('developmentInstall.verifyLaunchers')(function timeoutMs: 30_000, }); if (version.stdout.trim() !== `threadnote v${expectedVersion}`) { - return yield* Effect.fail(new Error('The managed CLI launcher did not execute the activated development release.')); + return yield* Effect.fail( + new ScriptError('The managed CLI launcher did not execute the activated development release.'), + ); } }); @@ -746,20 +759,20 @@ const requireCanonicalDevelopmentInstallRoots = Effect.fn('developmentInstall.re const current = yield* prepareCanonicalDevelopmentInstallRoots(installationRoot(path, system)); if (!platformPathEquals(path, system, current.installRoot, expectedInstallRoot)) { return yield* Effect.fail( - new Error('The managed Threadnote installation root changed or escaped its canonical location.'), + new ScriptError('The managed Threadnote installation root changed or escaped its canonical location.'), ); } if (!platformPathEquals(path, system, current.versionsRoot, expectedVersionsRoot)) { return yield* Effect.fail( - new Error('The managed Threadnote versions root changed or escaped its canonical location.'), + new ScriptError('The managed Threadnote versions root changed or escaped its canonical location.'), ); } if (path.basename(releaseRoot) !== version) { - return yield* Effect.fail(new Error('The development release name does not match its version.')); + return yield* Effect.fail(new ScriptError('The development release name does not match its version.')); } const releaseParent = yield* fs.realPath(path.dirname(releaseRoot)); if (!platformPathEquals(path, system, releaseParent, current.versionsRoot)) { - return yield* Effect.fail(new Error('The development release parent is not the canonical versions root.')); + return yield* Effect.fail(new ScriptError('The development release parent is not the canonical versions root.')); } if (Option.isSome(stagedRoot)) { const name = path.basename(stagedRoot.value); @@ -778,7 +791,9 @@ const requireCanonicalDevelopmentInstallRoots = Effect.fn('developmentInstall.re Option.isNone(canonical) || !platformPathEquals(path, system, canonical.value, path.join(stagedParent, name)) ) { - return yield* Effect.fail(new Error('The development staging directory changed or escaped before activation.')); + return yield* Effect.fail( + new ScriptError('The development staging directory changed or escaped before activation.'), + ); } } return current; @@ -802,13 +817,13 @@ function readPackageManifest(fs: FileSystem.FileSystem, file: string) { Effect.flatMap(source => Effect.try({ try: () => JSON.parse(source) as {readonly version?: unknown}, - catch: cause => new Error('Could not parse package.json.', {cause}), + catch: cause => new ScriptError('Could not parse package.json.', {cause}), }), ), Effect.flatMap(manifest => typeof manifest.version === 'string' && manifest.version.length > 0 ? Effect.succeed({version: manifest.version}) - : Effect.fail(new Error('package.json does not declare a version.')), + : Effect.fail(new ScriptError('package.json does not declare a version.')), ), ); } @@ -818,12 +833,12 @@ function readReleaseMetadata(fs: FileSystem.FileSystem, file: string) { Effect.flatMap(source => Effect.try({ try: () => JSON.parse(source) as unknown, - catch: cause => new Error('Could not parse the development release metadata.', {cause}), + catch: cause => new ScriptError('Could not parse the development release metadata.', {cause}), }), ), Effect.flatMap(value => { if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return Effect.fail(new Error('The development release metadata is invalid.')); + return Effect.fail(new ScriptError('The development release metadata is invalid.')); } const candidate = value as Partial<{ readonly executable: string; @@ -836,7 +851,7 @@ function readReleaseMetadata(fs: FileSystem.FileSystem, file: string) { typeof candidate.target === 'string' && typeof candidate.version === 'string' ? Effect.succeed(candidate as {executable: string; runtime: string; target: string; version: string}) - : Effect.fail(new Error('The development release metadata is incomplete.')); + : Effect.fail(new ScriptError('The development release metadata is incomplete.')); }), ); } @@ -849,4 +864,4 @@ const program = Effect.gen(function* () { return yield* installLocalStandalone(options); }); -if (import.meta.main) BunRuntime.runMain(program.pipe(Effect.provide(installerLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(program, installerLayer)); diff --git a/scripts/lint-file-length.ts b/scripts/lint-file-length.ts index 1c0a1e3d..3e196809 100644 --- a/scripts/lint-file-length.ts +++ b/scripts/lint-file-length.ts @@ -1,38 +1,23 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import {BunRuntime} from '@effect/platform-bun'; -import {Console, Effect} from 'effect'; -import {existsSync} from 'node:fs'; -import {join} from 'node:path'; -import {fileURLToPath} from 'node:url'; +import {Console, Effect, Result} from 'effect'; import {SystemInfo} from '../src/effect/system.js'; export const PRODUCTION_FILE_LINE_LIMIT = 2_000; export const PRODUCTION_CODE_ROOTS = ['src', 'website/src'] as const; -export const FILE_LENGTH_OXLINT_CONFIG = fileURLToPath(new URL('../.oxlintrc.max-lines.json', import.meta.url)); +export const FILE_LENGTH_OXLINT_CONFIG = Bun.fileURLToPath(new URL('../.oxlintrc.max-lines.json', import.meta.url)); const CODE_FILE_PATTERN = /\.(?:c|m)?(?:js|jsx|ts|tsx)$/u; const TEST_FILE_PATTERN = /\.(?:spec|test)\.(?:c|m)?(?:js|jsx|ts|tsx)$/u; const TEST_DIRECTORY_NAMES = new Set(['__tests__', 'test', 'tests']); -export type FileLengthSeverity = 'error' | 'warn'; - -export interface ProductionFileLintPartition { - readonly errorFiles: readonly string[]; - readonly warningFiles: readonly string[]; -} - -export interface ProductionFileLintPlan extends ProductionFileLintPartition { - readonly base: string | undefined; -} - export interface OxlintFileLengthRequest { readonly configPath: string; readonly files: readonly string[]; readonly repositoryRoot: string; - readonly severity: FileLengthSeverity; } export interface RunProductionFileLengthLintOptions { - readonly base?: string; readonly execute?: (request: OxlintFileLengthRequest) => number; readonly repositoryRoot: string; readonly roots?: readonly string[]; @@ -57,11 +42,13 @@ export function normalizeRepositoryPath(path: string): string | undefined { function normalizedProductionRoots(roots: readonly string[]): readonly string[] { const normalized = roots.map(root => normalizeRepositoryPath(root)); - if (normalized.some(root => root === undefined)) throw new Error('Production lint roots must be repository paths.'); + if (normalized.some(root => root === undefined)) { + throw new ScriptError('Production lint roots must be repository paths.'); + } const validRoots = normalized as string[]; for (const root of validRoots) { if (!(PRODUCTION_CODE_ROOTS as readonly string[]).includes(root)) { - throw new Error(`Unsupported production lint root: ${root}`); + throw new ScriptError(`Unsupported production lint root: ${root}`); } } return [...new Set(validRoots)].sort(comparePaths); @@ -70,111 +57,61 @@ function normalizedProductionRoots(roots: readonly string[]): readonly string[] export function isProductionCodePath(path: string, roots: readonly string[] = PRODUCTION_CODE_ROOTS): boolean { const normalized = normalizeRepositoryPath(path); if (!normalized || !CODE_FILE_PATTERN.test(normalized) || TEST_FILE_PATTERN.test(normalized)) return false; - - const segments = normalized.split('/'); - if (segments.some(segment => TEST_DIRECTORY_NAMES.has(segment))) return false; - + if (normalized.split('/').some(segment => TEST_DIRECTORY_NAMES.has(segment))) return false; return roots.some(root => normalized === root || normalized.startsWith(`${root}/`)); } -export function partitionProductionCodeFiles( +/** Return every unique production source path, independent of Git change state or iteration order. */ +export function productionCodeFiles( files: Iterable, - changedPaths: Iterable, roots: readonly string[] = PRODUCTION_CODE_ROOTS, -): ProductionFileLintPartition { - const changed = new Set( - [...changedPaths].map(path => normalizeRepositoryPath(path)).filter((path): path is string => path !== undefined), - ); +): readonly string[] { const productionFiles = new Set(); for (const file of files) { const normalized = normalizeRepositoryPath(file); if (normalized && isProductionCodePath(normalized, roots)) productionFiles.add(normalized); } - - const errorFiles: string[] = []; - const warningFiles: string[] = []; - for (const file of [...productionFiles].sort(comparePaths)) { - (changed.has(file) ? errorFiles : warningFiles).push(file); - } - return {errorFiles, warningFiles}; -} - -function decodeOutput(output: Uint8Array | undefined): string { - return output ? new TextDecoder().decode(output) : ''; + return [...productionFiles].sort(comparePaths); } function decodeNullSeparated(output: Uint8Array | undefined): readonly string[] { - return decodeOutput(output).split('\0').filter(Boolean); + return output ? new TextDecoder().decode(output).split('\0').filter(Boolean) : []; } -function runGit(repositoryRoot: string, arguments_: readonly string[]): ReturnType { - return Bun.spawnSync({ +function gitPaths(repositoryRoot: string, arguments_: readonly string[]): readonly string[] { + const result = Bun.spawnSync({ cmd: ['git', ...arguments_], cwd: repositoryRoot, stderr: 'pipe', stdout: 'pipe', }); -} - -function gitPaths(repositoryRoot: string, arguments_: readonly string[]): readonly string[] { - const result = runGit(repositoryRoot, arguments_); if (result.exitCode !== 0) { - const detail = decodeOutput(result.stderr).trim(); - throw new Error(`git ${arguments_.join(' ')} failed${detail ? `: ${detail}` : '.'}`); + const detail = result.stderr ? new TextDecoder().decode(result.stderr).trim() : ''; + throw new ScriptError(`git ${arguments_.join(' ')} failed${detail ? `: ${detail}` : '.'}`); } return decodeNullSeparated(result.stdout); } -function commitExists(repositoryRoot: string, reference: string): boolean { - if (reference.startsWith('-') || reference.includes('\0')) return false; - return runGit(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${reference}^{commit}`]).exitCode === 0; -} - -export function resolveProductionLintBase(repositoryRoot: string, requestedBase?: string): string | undefined { - const base = requestedBase?.trim(); - if (base) { - if (!commitExists(repositoryRoot, base)) throw new Error(`Production file lint base is not a commit: ${base}`); - return base; - } - return commitExists(repositoryRoot, 'origin/main') ? 'origin/main' : undefined; -} - -export function collectProductionFileLintPlan( +export function collectProductionCodeFiles( repositoryRoot: string, - options: {readonly base?: string; readonly roots?: readonly string[]} = {}, -): ProductionFileLintPlan { - const roots = normalizedProductionRoots(options.roots ?? PRODUCTION_CODE_ROOTS); - const pathspec = ['--', ...roots]; - const base = resolveProductionLintBase(repositoryRoot, options.base); - const allFiles = gitPaths(repositoryRoot, [ + roots: readonly string[] = PRODUCTION_CODE_ROOTS, +): readonly string[] { + const normalizedRoots = normalizedProductionRoots(roots); + const pathspec = ['--', ...normalizedRoots]; + const deletedFiles = new Set(gitPaths(repositoryRoot, ['ls-files', '--deleted', '-z', ...pathspec])); + const files = gitPaths(repositoryRoot, [ 'ls-files', '--cached', '--others', '--exclude-standard', '-z', ...pathspec, - ]).filter(path => existsSync(join(repositoryRoot, path))); - const changedPaths = new Set([ - ...gitPaths(repositoryRoot, ['diff', '--name-only', '--no-renames', '-z', 'HEAD', ...pathspec]), - ...gitPaths(repositoryRoot, ['ls-files', '--others', '--exclude-standard', '-z', ...pathspec]), - ]); - if (base) { - for (const path of gitPaths(repositoryRoot, [ - 'diff', - '--name-only', - '--no-renames', - '-z', - `${base}...HEAD`, - ...pathspec, - ])) { - changedPaths.add(path); - } - } - return {...partitionProductionCodeFiles(allFiles, changedPaths, roots), base}; + ]).filter(path => !deletedFiles.has(path)); + return productionCodeFiles(files, normalizedRoots); } function executeOxlint(request: OxlintFileLengthRequest): number { - const severityArguments = request.severity === 'error' ? ['--deny', 'max-lines'] : []; + if (request.files.length === 0) return 0; const result = Bun.spawnSync({ cmd: [ process.execPath, @@ -186,7 +123,9 @@ function executeOxlint(request: OxlintFileLengthRequest): number { '--disable-oxc-plugin', '--disable-typescript-plugin', '--no-error-on-unmatched-pattern', - ...severityArguments, + '--threads=1', + '--deny-warnings', + '--report-unused-disable-directives-severity=error', ...request.files, ], cwd: request.repositoryRoot, @@ -197,72 +136,42 @@ function executeOxlint(request: OxlintFileLengthRequest): number { } export function runProductionFileLengthLint(options: RunProductionFileLengthLintOptions): number { - const plan = collectProductionFileLintPlan(options.repositoryRoot, { - base: options.base, - roots: options.roots, + const files = collectProductionCodeFiles(options.repositoryRoot, options.roots); + return (options.execute ?? executeOxlint)({ + configPath: FILE_LENGTH_OXLINT_CONFIG, + files, + repositoryRoot: options.repositoryRoot, }); - const execute = options.execute ?? executeOxlint; - for (const request of [ - {files: plan.warningFiles, severity: 'warn' as const}, - {files: plan.errorFiles, severity: 'error' as const}, - ]) { - if (request.files.length === 0) continue; - const exitCode = execute({ - configPath: FILE_LENGTH_OXLINT_CONFIG, - files: request.files, - repositoryRoot: options.repositoryRoot, - severity: request.severity, - }); - if (exitCode !== 0) return exitCode; - } - return 0; } -function parseArguments(arguments_: readonly string[]): {readonly base?: string; readonly roots: readonly string[]} { - const roots: string[] = []; - let base: string | undefined; - for (let index = 0; index < arguments_.length; index += 1) { - const argument = arguments_[index]; - if (argument === '--base') { - base = arguments_[index + 1]; - if (!base) throw new Error('--base requires a Git commit or reference.'); - index += 1; - continue; - } - if (argument.startsWith('-')) throw new Error(`Unknown production file lint option: ${argument}`); - roots.push(argument); +function parseArguments(arguments_: readonly string[]): readonly string[] { + if (arguments_.some(argument => argument.startsWith('-'))) { + throw new ScriptError('Production file lint accepts only optional production roots.'); } - return {base, roots: normalizedProductionRoots(roots.length === 0 ? PRODUCTION_CODE_ROOTS : roots)}; + return normalizedProductionRoots(arguments_.length === 0 ? PRODUCTION_CODE_ROOTS : arguments_); } if (import.meta.main) { BunRuntime.runMain( - Effect.gen(function* () { - const system = yield* SystemInfo; - const outcome = yield* Effect.try({ - try: () => { - const {base, roots} = parseArguments(Bun.argv.slice(2)); - return runProductionFileLengthLint({ - base: base ?? process.env.THREADNOTE_LINT_BASE, - repositoryRoot: fileURLToPath(new URL('..', import.meta.url)), - roots, - }); - }, - catch: cause => cause, - }).pipe( - Effect.match({ - onFailure: cause => ({cause, success: false}) as const, - onSuccess: exitCode => ({exitCode, success: true}) as const, - }), - ); - if (!outcome.success) { - yield* Console.error( - `Production file length lint failed: ${outcome.cause instanceof Error ? outcome.cause.message : String(outcome.cause)}`, - ); - system.setExitCode(2); - return; - } - system.setExitCode(outcome.exitCode); - }).pipe(Effect.provide(SystemInfo.layer)), + provideScriptLayer( + Effect.gen(function* () { + const system = yield* SystemInfo; + const outcome = yield* Effect.try({ + try: () => + runProductionFileLengthLint({ + repositoryRoot: Bun.fileURLToPath(new URL('..', import.meta.url)), + roots: parseArguments(Bun.argv.slice(2)), + }), + catch: cause => scriptError(cause, 'Could not evaluate the production file-length policy.'), + }).pipe(Effect.result); + if (Result.isFailure(outcome)) { + yield* Console.error(`Production file length lint failed: ${outcome.failure.message}`); + system.setExitCode(2); + return; + } + system.setExitCode(outcome.success); + }), + SystemInfo.layer, + ), ); } diff --git a/scripts/lint.ts b/scripts/lint.ts index 81704558..18109e67 100644 --- a/scripts/lint.ts +++ b/scripts/lint.ts @@ -1,66 +1,16 @@ -const LINT_TARGETS = ['config/lint', 'scripts', 'src', 'test', 'website/src', 'website/vite.config.ts']; -const LINTABLE_EXTENSION = /\.(?:[cm]?[jt]sx?)$/; -// Everything through the Workset Search 2.0 release at this commit predates the lint ratchet and remains warning-only. -// Once a configured CI base contains it, that newer base becomes the strict boundary. -const LINT_ADOPTION_BASE = 'ce63d995f5e5685246f6866ab889a54fd70b5322'; -const decoder = new TextDecoder(); +import {ScriptError} from './effect/errors.js'; -function gitLines(arguments_: readonly string[], allowFailure = false): readonly string[] { - const result = Bun.spawnSync({ - cmd: ['git', ...arguments_], - stderr: 'pipe', - stdout: 'pipe', - }); - if (result.exitCode !== 0) { - if (allowFailure) return []; - throw new Error(decoder.decode(result.stderr).trim() || `git ${arguments_.join(' ')} failed`); - } - return decoder - .decode(result.stdout) - .split('\n') - .map(path => path.trim()) - .filter(Boolean); -} +export const LINT_TARGETS = ['config/lint', 'scripts', 'src', 'test', 'website/src', 'website/vite.config.ts'] as const; -function gitSucceeds(arguments_: readonly string[]): boolean { - return ( - Bun.spawnSync({ - cmd: ['git', ...arguments_], - stderr: 'ignore', - stdout: 'ignore', - }).exitCode === 0 - ); -} - -export function normalizeChangedLintPaths(paths: readonly string[]): readonly string[] { - return [ - ...new Set(paths.map(path => path.replaceAll('\\', '/')).filter(path => LINTABLE_EXTENSION.test(path))), - ].sort(); -} - -async function changedLintFiles(): Promise { - const paths = [ - ...gitLines(['diff', '--name-only', '--diff-filter=ACMR', 'HEAD', '--', ...LINT_TARGETS]), - ...gitLines(['ls-files', '--others', '--exclude-standard', '--', ...LINT_TARGETS]), - ]; - const configuredBase = process.env.LINT_BASE; - const requestedBase = configuredBase && !/^0+$/.test(configuredBase) ? configuredBase : undefined; - const base = - requestedBase && - gitSucceeds(['cat-file', '-e', `${LINT_ADOPTION_BASE}^{commit}`]) && - gitSucceeds(['merge-base', '--is-ancestor', requestedBase, LINT_ADOPTION_BASE]) - ? LINT_ADOPTION_BASE - : requestedBase; - if (base) { - paths.push(...gitLines(['diff', '--name-only', '--diff-filter=ACMR', `${base}...HEAD`, '--', ...LINT_TARGETS])); - } - - const normalized = normalizeChangedLintPaths(paths); - const existing = await Promise.all( - normalized.map(async path => ((await Bun.file(path).exists()) ? path : undefined)), - ); - return existing.filter((path): path is string => path !== undefined); -} +export const STRICT_LINT_ARGUMENTS = [ + '--config', + '.oxlintrc.strict.json', + '--threads=1', + '--deny-warnings', + '--report-unused-disable-directives-severity=error', + '--ignore-pattern', + 'test/evaluation/fixtures/**/repository/**', +] as const; function runOxlint(arguments_: readonly string[]): number { const child = Bun.spawnSync({ @@ -71,14 +21,15 @@ function runOxlint(arguments_: readonly string[]): number { return child.exitCode; } -export async function lint(): Promise { - const warningExit = runOxlint(LINT_TARGETS); - if (warningExit !== 0) return warningExit; - - const changed = await changedLintFiles(); - if (changed.length === 0) return 0; - process.stdout.write(`Strict lint: ${changed.length} changed file${changed.length === 1 ? '' : 's'}\n`); - return runOxlint(['--config', '.oxlintrc.strict.json', ...changed]); +/** Run one deterministic, full-repository lint policy with no warning-only grandfathering. */ +export function lint(execute: (arguments_: readonly string[]) => number = runOxlint): number { + return execute([...STRICT_LINT_ARGUMENTS, ...LINT_TARGETS]); } -if (import.meta.main) process.exitCode = await lint(); +if (import.meta.main) { + try { + process.exitCode = lint(); + } catch (cause) { + throw new ScriptError('Could not run the repository lint policy.', {cause}); + } +} diff --git a/scripts/recall-vector-storage-budget.ts b/scripts/recall-vector-storage-budget.ts index a1414b13..69a34ba7 100644 --- a/scripts/recall-vector-storage-budget.ts +++ b/scripts/recall-vector-storage-budget.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {Database} from 'bun:sqlite'; const MEBIBYTE = 1024 * 1024; @@ -34,7 +35,7 @@ export function assessVectorDatabaseStorage( incremental: VectorDatabaseStorageMeasurement, ): VectorDatabaseStorageBudget { if (!Number.isSafeInteger(documents) || documents <= 0) { - throw new Error('Vector database storage budget requires a positive document count.'); + throw new ScriptError('Vector database storage budget requires a positive document count.'); } assertMeasurement(initial); assertMeasurement(incremental); @@ -61,18 +62,18 @@ function assertMeasurement(measurement: VectorDatabaseStorageMeasurement): void !Number.isSafeInteger(measurement.databaseBytes) || measurement.databaseBytes < 0 ) { - throw new Error('Invalid vector database storage measurement.'); + throw new ScriptError('Invalid vector database storage measurement.'); } } function safeProduct(left: number, right: number, label: string): number { const value = left * right; - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid ${label}.`); + if (!Number.isSafeInteger(value) || value < 0) throw new ScriptError(`Invalid ${label}.`); return value; } function safeSum(left: number, right: number, label: string): number { const value = left + right; - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid ${label}.`); + if (!Number.isSafeInteger(value) || value < 0) throw new ScriptError(`Invalid ${label}.`); return value; } diff --git a/scripts/site-doc-pages.ts b/scripts/site-doc-pages.ts index 364ab40e..c4b1d61d 100644 --- a/scripts/site-doc-pages.ts +++ b/scripts/site-doc-pages.ts @@ -1,5 +1,7 @@ -import {mkdir, readFile, writeFile} from 'node:fs/promises'; -import {dirname, join} from 'node:path'; +import {provideScriptLayer, ScriptError} from './effect/errors.js'; +import * as BunRuntime from '@effect/platform-bun/BunRuntime'; +import * as BunServices from '@effect/platform-bun/BunServices'; +import {Console, Effect, FileSystem, Path} from 'effect'; import {docsSections, type DocsArticle} from '../website/src/content/docs.js'; import {docsArticlePath} from '../website/src/lib/routes.js'; @@ -35,11 +37,12 @@ function replaceTagAttribute( 'i', ); const tag = html.match(tagPattern)?.[0]; - if (!tag) throw new Error(`Docs HTML template is missing ${tagName}[${identifyingAttribute}="${identifyingValue}"]`); + if (!tag) + throw new ScriptError(`Docs HTML template is missing ${tagName}[${identifyingAttribute}="${identifyingValue}"]`); const attributePattern = new RegExp(`\\b${escapeRegExp(updatedAttribute)}="[^"]*"`, 'i'); if (!attributePattern.test(tag)) { - throw new Error( + throw new ScriptError( `Docs HTML template ${tagName}[${identifyingAttribute}="${identifyingValue}"] is missing ${updatedAttribute}`, ); } @@ -63,7 +66,7 @@ export function renderDocsArticleHtml( html = replaceTagAttribute(html, 'meta', 'property', 'og:url', 'content', canonicalUrl); html = replaceTagAttribute(html, 'meta', 'name', 'twitter:title', 'content', pageTitle); html = replaceTagAttribute(html, 'meta', 'name', 'twitter:description', 'content', article.summary); - if (!/[^<]*<\/title>/i.test(html)) throw new Error('Docs HTML template is missing its title'); + if (!/<title>[^<]*<\/title>/i.test(html)) throw new ScriptError('Docs HTML template is missing its title'); return html.replace(/<title>[^<]*<\/title>/i, `<title>${escapeHtml(pageTitle)}`); } @@ -77,36 +80,48 @@ export function renderDocsSitemap(sitemap: string, articles: readonly Pick')) throw new Error('Website sitemap is missing '); + if (!sitemap.includes('')) throw new ScriptError('Website sitemap is missing '); return sitemap.replace('', `${generated}\n`); } -export async function generateDocsArticlePages(siteDist = join(process.cwd(), 'site-dist')): Promise { - const templatePath = join(siteDist, 'docs', 'index.html'); - const [template, sitemap] = await Promise.all([ - readFile(templatePath, 'utf8'), - readFile(join(siteDist, 'sitemap.xml'), 'utf8'), - ]); +export const generateDocsArticlePages = Effect.fn('siteDocs.generateArticlePages')(function* (siteDist?: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const outputRoot = siteDist ?? path.resolve(process.cwd(), 'site-dist'); + const templatePath = path.join(outputRoot, 'docs', 'index.html'); + const [template, sitemap] = yield* Effect.all( + [fs.readFileString(templatePath), fs.readFileString(path.join(outputRoot, 'sitemap.xml'))], + {concurrency: 2}, + ); const articleIds = new Set(); - await Promise.all( - docsArticles.map(async article => { - if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(article.id)) { - throw new Error(`Docs article id is not a URL-safe slug: ${article.id}`); - } - if (articleIds.has(article.id)) throw new Error(`Duplicate docs article id: ${article.id}`); - articleIds.add(article.id); - const outputPath = join(siteDist, docsArticlePath(article.id), 'index.html'); - await mkdir(dirname(outputPath), {recursive: true}); - await writeFile(outputPath, renderDocsArticleHtml(template, article)); - }), + yield* Effect.forEach( + docsArticles, + article => + Effect.gen(function* () { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(article.id)) { + throw new ScriptError(`Docs article id is not a URL-safe slug: ${article.id}`); + } + if (articleIds.has(article.id)) throw new ScriptError(`Duplicate docs article id: ${article.id}`); + articleIds.add(article.id); + const outputPath = path.join(outputRoot, docsArticlePath(article.id), 'index.html'); + yield* fs.makeDirectory(path.dirname(outputPath), {recursive: true}); + yield* fs.writeFileString(outputPath, renderDocsArticleHtml(template, article)); + }), + {concurrency: 1}, ); - await writeFile(join(siteDist, 'sitemap.xml'), renderDocsSitemap(sitemap, docsArticles)); + yield* fs.writeFileString(path.join(outputRoot, 'sitemap.xml'), renderDocsSitemap(sitemap, docsArticles)); return docsArticles.length; -} +}); if (import.meta.main) { - const generatedCount = await generateDocsArticlePages(); - process.stdout.write(`Generated ${generatedCount} crawler-visible documentation pages.\n`); + BunRuntime.runMain( + provideScriptLayer( + generateDocsArticlePages().pipe( + Effect.tap(generatedCount => Console.log(`Generated ${generatedCount} crawler-visible documentation pages.`)), + ), + BunServices.layer, + ), + ); } diff --git a/scripts/site-performance-evidence.ts b/scripts/site-performance-evidence.ts index 1bcfba83..ffd2da5c 100644 --- a/scripts/site-performance-evidence.ts +++ b/scripts/site-performance-evidence.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; import {parseBenchmarkArtifactV1} from '../src/evaluation/benchmark.js'; import { pendingPerformanceEvidence, @@ -17,7 +18,7 @@ export function performanceArtifactPublicUrl(siteBase: string): string { siteBase.includes('//') || segments.some(segment => segment === '.' || segment === '..' || !/^[A-Za-z0-9._~-]+$/.test(segment)) ) { - throw new Error('THREADNOTE_SITE_BASE must be a root-relative directory path ending in /.'); + throw new ScriptError('THREADNOTE_SITE_BASE must be a root-relative directory path ending in /.'); } return `${siteBase}performance-evidence.json`; } @@ -47,13 +48,13 @@ const utcTimestampPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/ function exactRecord(value: unknown, path: string, keys: readonly string[]): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`Performance binding ${path} must be an object.`); + throw new ScriptError(`Performance binding ${path} must be an object.`); } const record = value as Record; const actualKeys = Object.keys(record).sort(); const expectedKeys = [...keys].sort(); if (actualKeys.length !== expectedKeys.length || actualKeys.some((key, index) => key !== expectedKeys[index])) { - throw new Error(`Performance binding ${path} has unexpected or missing fields.`); + throw new ScriptError(`Performance binding ${path} has unexpected or missing fields.`); } return record; } @@ -67,7 +68,7 @@ function matchingString( ): string { const value = record[key]; if (typeof value !== 'string' || !pattern.test(value)) { - throw new Error(`Performance binding ${path}.${key} must be ${label}.`); + throw new ScriptError(`Performance binding ${path}.${key} must be ${label}.`); } return value; } @@ -80,7 +81,7 @@ export function validatePerformanceArtifactBinding(input: unknown): PerformanceA 'sourceThreadnoteCommit', 'sourceTreeSha256', ]); - if (binding.schemaVersion !== 1) throw new Error('Performance binding root.schemaVersion must be 1.'); + if (binding.schemaVersion !== 1) throw new ScriptError('Performance binding root.schemaVersion must be 1.'); matchingString(binding, 'artifactSha256', 'root', sha256Pattern, 'a lowercase SHA-256 digest'); matchingString(binding, 'generatedAt', 'root', utcTimestampPattern, 'an ISO-8601 UTC timestamp'); matchingString(binding, 'sourceThreadnoteCommit', 'root', sha40Pattern, 'a lowercase 40-character Git commit'); @@ -99,7 +100,7 @@ function parseRetainedPerformanceArtifactBytes(artifactBytes: Uint8Array): Retai try { parsed = JSON.parse(new TextDecoder().decode(artifactBytes)); } catch { - throw new Error('Retained performance artifact is not valid JSON.'); + throw new ScriptError('Retained performance artifact is not valid JSON.'); } parseBenchmarkArtifactV1(parsed); return validateRetainedPerformancePayload(parsed); @@ -115,21 +116,21 @@ export function bindRetainedPerformanceArtifact(input: { }): PerformanceEvidence { const binding = validatePerformanceArtifactBinding(input.binding); if (!sha256Pattern.test(input.currentSourceTreeSha256)) { - throw new Error('Current performance source-tree digest is invalid.'); + throw new ScriptError('Current performance source-tree digest is invalid.'); } const actualArtifactSha256 = sha256Hex(input.artifactBytes); if (actualArtifactSha256 !== binding.artifactSha256) { - throw new Error( + throw new ScriptError( `Retained performance artifact SHA-256 mismatch: expected ${binding.artifactSha256}, got ${actualArtifactSha256}.`, ); } if (input.currentSourceTreeSha256 !== binding.sourceTreeSha256) { - throw new Error('Retained performance evidence does not match the current Threadnote source tree.'); + throw new ScriptError('Retained performance evidence does not match the current Threadnote source tree.'); } const payload = parseRetainedPerformanceArtifactBytes(input.artifactBytes); if (payload.environment.commit !== binding.sourceThreadnoteCommit) { - throw new Error('Retained performance artifact and binding name different Threadnote source commits.'); + throw new ScriptError('Retained performance artifact and binding name different Threadnote source commits.'); } const artifact = retainedPerformanceArtifactFromHarness(payload, { artifactUrl: input.artifactPublicUrl, @@ -156,27 +157,29 @@ function runGit(repositoryRoot: string, arguments_: readonly string[]): ReturnTy function requireSuccessfulGit(result: ReturnType, operation: string): void { if (result.exitCode !== 0) { - throw new Error(`Could not ${operation}: ${decodeOutput(result.stderr) || `git exited with ${result.exitCode}`}.`); + throw new ScriptError( + `Could not ${operation}: ${decodeOutput(result.stderr) || `git exited with ${result.exitCode}`}.`, + ); } } export function assertPerformanceSourceClean(repositoryRoot: string): void { const unstaged = runGit(repositoryRoot, ['diff', '--quiet', '--', ...performanceSourcePathspecs]); if (unstaged.exitCode === 1) { - throw new Error('Performance-bound sources contain tracked working-tree modifications.'); + throw new ScriptError('Performance-bound sources contain tracked working-tree modifications.'); } requireSuccessfulGit(unstaged, 'inspect tracked performance-source modifications'); const staged = runGit(repositoryRoot, ['diff', '--cached', '--quiet', '--', ...performanceSourcePathspecs]); if (staged.exitCode === 1) { - throw new Error('Performance-bound sources contain staged modifications.'); + throw new ScriptError('Performance-bound sources contain staged modifications.'); } requireSuccessfulGit(staged, 'inspect staged performance-source modifications'); const untracked = runGit(repositoryRoot, ['ls-files', '--others', '-z', '--', ...performanceSourcePathspecs]); requireSuccessfulGit(untracked, 'inspect untracked performance sources'); if ((untracked.stdout?.byteLength ?? 0) > 0) { - throw new Error('Performance-bound sources contain untracked files.'); + throw new ScriptError('Performance-bound sources contain untracked files.'); } } @@ -184,13 +187,15 @@ function verifySourceCommit(repositoryRoot: string, sourceCommit: string): void assertPerformanceSourceClean(repositoryRoot); const commit = runGit(repositoryRoot, ['cat-file', '-e', `${sourceCommit}^{commit}`]); if (commit.exitCode !== 0) { - throw new Error( + throw new ScriptError( `Retained performance source commit ${sourceCommit} is unavailable; use a full Git checkout for the website build.`, ); } const ancestor = runGit(repositoryRoot, ['merge-base', '--is-ancestor', sourceCommit, 'HEAD']); if (ancestor.exitCode !== 0) { - throw new Error(`Retained performance source commit ${sourceCommit} is not an ancestor of the website build.`); + throw new ScriptError( + `Retained performance source commit ${sourceCommit} is not an ancestor of the website build.`, + ); } const changed = runGit(repositoryRoot, [ 'diff', @@ -201,7 +206,9 @@ function verifySourceCommit(repositoryRoot: string, sourceCommit: string): void ...performanceSourcePathspecs, ]); if (changed.exitCode !== 0) { - throw new Error('Threadnote runtime sources changed after the retained performance run; publish fresh evidence.'); + throw new ScriptError( + 'Threadnote runtime sources changed after the retained performance run; publish fresh evidence.', + ); } } @@ -210,7 +217,7 @@ function verifyReleaseEvidenceSource(repositoryRoot: string, payload: RetainedPe const resolved = runGit(repositoryRoot, ['rev-parse', '--verify', `${ref}^{commit}`]); requireSuccessfulGit(resolved, 'resolve the retained performance release tag'); if (decodeOutput(resolved.stdout) !== payload.environment.commit) { - throw new Error('Retained performance release tag does not resolve to the measured Threadnote commit.'); + throw new ScriptError('Retained performance release tag does not resolve to the measured Threadnote commit.'); } } @@ -218,7 +225,7 @@ export async function computePerformanceSourceTreeSha256(repositoryRoot: string) assertPerformanceSourceClean(repositoryRoot); const listed = runGit(repositoryRoot, ['ls-files', '--stage', '-z', '--', ...performanceSourcePathspecs]); if (listed.exitCode !== 0) { - throw new Error(`Could not inventory performance-bound sources: ${decodeOutput(listed.stderr)}.`); + throw new ScriptError(`Could not inventory performance-bound sources: ${decodeOutput(listed.stderr)}.`); } const entries = new TextDecoder() .decode(listed.stdout) @@ -226,15 +233,15 @@ export async function computePerformanceSourceTreeSha256(repositoryRoot: string) .filter(Boolean) .map(entry => { const tabIndex = entry.indexOf('\t'); - if (tabIndex === -1) throw new Error('Git returned an invalid performance source entry.'); + if (tabIndex === -1) throw new ScriptError('Git returned an invalid performance source entry.'); const metadata = entry.slice(0, tabIndex).split(' '); const mode = metadata[0]; const path = entry.slice(tabIndex + 1); - if (!mode || !path) throw new Error('Git returned an incomplete performance source entry.'); + if (!mode || !path) throw new ScriptError('Git returned an incomplete performance source entry.'); return {mode, path}; }) .sort((left, right) => left.path.localeCompare(right.path)); - if (entries.length === 0) throw new Error('Performance source inventory is empty.'); + if (entries.length === 0) throw new ScriptError('Performance source inventory is empty.'); const hasher = new Bun.CryptoHasher('sha256'); for (const entry of entries) { @@ -273,14 +280,14 @@ export async function loadRetainedPerformanceEvidence( ); } if (!artifactExists || !bindingExists) { - throw new Error('Retained performance evidence requires both the local JSON artifact and its binding file.'); + throw new ScriptError('Retained performance evidence requires both the local JSON artifact and its binding file.'); } let bindingInput: unknown; try { bindingInput = JSON.parse(await bindingFile.text()); } catch { - throw new Error('Retained performance binding is not valid JSON.'); + throw new ScriptError('Retained performance binding is not valid JSON.'); } const binding = validatePerformanceArtifactBinding(bindingInput); verifySourceCommit(repositoryRoot, binding.sourceThreadnoteCommit); @@ -304,7 +311,9 @@ export async function loadRetainedPerformanceEvidence( export async function writePerformanceArtifactBinding(repositoryRoot: string): Promise { const artifactFile = Bun.file(`${repositoryRoot}/${performanceArtifactRelativePath}`); if (!(await artifactFile.exists())) { - throw new Error(`Place the reviewed payload at ${performanceArtifactRelativePath} before creating its binding.`); + throw new ScriptError( + `Place the reviewed payload at ${performanceArtifactRelativePath} before creating its binding.`, + ); } const artifactBytes = new Uint8Array(await artifactFile.arrayBuffer()); diff --git a/scripts/site-release-notes.ts b/scripts/site-release-notes.ts index 500e3d8c..4d182292 100644 --- a/scripts/site-release-notes.ts +++ b/scripts/site-release-notes.ts @@ -1,3 +1,4 @@ +import {ScriptError} from './effect/errors.js'; export interface StableReleaseVersion { readonly version: string; readonly major: number; @@ -104,7 +105,7 @@ function runGit(repositoryRoot: string, arguments_: readonly string[]): string { }); if (result.exitCode !== 0) { const detail = result.stderr.toString().trim(); - throw new Error(`Could not load website release notes${detail ? `: ${detail}` : '.'}`); + throw new ScriptError(`Could not load website release notes${detail ? `: ${detail}` : '.'}`); } return result.stdout.toString(); } @@ -163,16 +164,17 @@ export function loadLatestMajorWebsiteReleases(repositoryRoot: string): readonly const refs = includePreparedWebsiteRelease(published, loadPreparedWebsiteRelease(repositoryRoot, published)); const selected = selectLatestMajorReleases(refs); - if (selected.length === 0) throw new Error('The website needs at least one published or prepared stable release.'); + if (selected.length === 0) + throw new ScriptError('The website needs at least one published or prepared stable release.'); const sourcesByVersion = new Map(refs.map(release => [release.version, release])); return selected.map(release => { const releaseNotePath = `.github/release-notes/${release.version}.md`; const source = sourcesByVersion.get(release.version); - if (source === undefined) throw new Error(`Could not resolve website release source for ${release.version}.`); + if (source === undefined) throw new ScriptError(`Could not resolve website release source for ${release.version}.`); const markdown = runGit(repositoryRoot, ['show', `${source.noteRef}:${releaseNotePath}`]); const {summary, highlights} = summarizeReleaseNote(markdown); - if (!summary) throw new Error(`${releaseNotePath} needs an introductory release summary.`); + if (!summary) throw new ScriptError(`${releaseNotePath} needs an introductory release summary.`); return { ...release, highlights, diff --git a/scripts/smoke-self-contained.ts b/scripts/smoke-self-contained.ts index ee043de0..72bb5a19 100644 --- a/scripts/smoke-self-contained.ts +++ b/scripts/smoke-self-contained.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, scriptError, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Client} from '@modelcontextprotocol/sdk/client/index.js'; @@ -19,7 +20,7 @@ const smokeSelfContained = Effect.scoped( const root = yield* path.fromFileUrl(ROOT_URL); const executable = path.join(root, 'dist', process.platform === 'win32' ? 'threadnote.exe' : 'threadnote'); if (!(yield* fs.exists(executable))) { - return yield* Effect.fail(new Error('Standalone executable is missing; run bun run build first.')); + return yield* Effect.fail(new ScriptError('Standalone executable is missing; run bun run build first.')); } const temporaryRoot = yield* fs.makeTempDirectoryScoped({prefix: 'threadnote-standalone-smoke-'}); @@ -31,7 +32,7 @@ const smokeSelfContained = Effect.scoped( } const gitExecutable = Option.fromNullishOr(Bun.which('git')); if (Option.isNone(gitExecutable)) { - return yield* Effect.fail(new Error('Standalone graph smoke requires Git on the build host.')); + return yield* Effect.fail(new ScriptError('Standalone graph smoke requires Git on the build host.')); } yield* writePolyglotRepository(fs, path, invocationDirectory, gitExecutable.value); @@ -56,14 +57,14 @@ const smokeSelfContained = Effect.scoped( const version = yield* run(['--version']); if (!/threadnote v4\./.test(version)) { - return yield* Effect.fail(new Error(`Standalone release reported an unexpected version:\n${version}`)); + return yield* Effect.fail(new ScriptError(`Standalone release reported an unexpected version:\n${version}`)); } yield* run(['install', '--dry-run', '--no-start']); const runtime = yield* run(['models', 'runtime']); if (!/node-llama-cpp:\s+prebuilt/i.test(runtime)) { - return yield* Effect.fail(new Error(`Native runtime was not loaded from the release payload:\n${runtime}`)); + return yield* Effect.fail(new ScriptError(`Native runtime was not loaded from the release payload:\n${runtime}`)); } yield* run([ @@ -79,7 +80,7 @@ const smokeSelfContained = Effect.scoped( ]); const recall = yield* run(['recall', '--query', `${SMOKE_MARKER} standalone recall`]); if (!recall.includes('standalone-bun-smoke.md')) { - return yield* Effect.fail(new Error(`Standalone lexical recall missed the stored memory:\n${recall}`)); + return yield* Effect.fail(new ScriptError(`Standalone lexical recall missed the stored memory:\n${recall}`)); } const storedMemoryPath = path.join( @@ -111,7 +112,7 @@ const smokeSelfContained = Effect.scoped( }); if (largeRead.bytes <= 65_536 || largeRead.hasStart !== true || largeRead.hasEnd !== true) { return yield* Effect.fail( - new Error( + new ScriptError( `Standalone large memory read was truncated before stdout drained: ${largeRead.bytes} bytes; ` + `start=${largeRead.hasStart}; end=${largeRead.hasEnd}.`, ), @@ -121,17 +122,17 @@ const smokeSelfContained = Effect.scoped( const lexicalDatabase = path.join(threadnoteHome, 'indexes', 'lexical', 'active-v3.sqlite'); const lexicalInfo = yield* fs.stat(lexicalDatabase); if (lexicalInfo.type !== 'File' || lexicalInfo.size <= 0) { - return yield* Effect.fail(new Error('Standalone recall did not create a populated Bun SQLite index.')); + return yield* Effect.fail(new ScriptError('Standalone recall did not create a populated Bun SQLite index.')); } const doctor = yield* run(['doctor', '--dry-run']); if (!/bun runtime:\s+v1\.3\.14;\s+embedded/i.test(doctor) || /Node runtime/i.test(doctor)) { - return yield* Effect.fail(new Error(`Doctor did not report the embedded Bun runtime:\n${doctor}`)); + return yield* Effect.fail(new ScriptError(`Doctor did not report the embedded Bun runtime:\n${doctor}`)); } const indexed = yield* run(['graph', 'index']); if (!/14 symbols|symbols/i.test(indexed)) { - return yield* Effect.fail(new Error(`Standalone graph index did not complete:\n${indexed}`)); + return yield* Effect.fail(new ScriptError(`Standalone graph index did not complete:\n${indexed}`)); } const graphOperations = yield* Effect.all( [ @@ -145,7 +146,7 @@ const smokeSelfContained = Effect.scoped( for (const [index, expected] of ['java', 'KotlinApp', 'typescriptHelper', 'swiftBoot'].entries()) { if (!graphOperations[index]!.includes(expected)) { return yield* Effect.fail( - new Error(`Standalone graph operation ${index + 1} missed ${expected}:\n${graphOperations[index]}`), + new ScriptError(`Standalone graph operation ${index + 1} missed ${expected}:\n${graphOperations[index]}`), ); } } @@ -197,7 +198,7 @@ const readLargeOutputThroughPlatformPipe = Effect.fn('smokeSelfContained.readLar readonly hasEnd: boolean; readonly hasStart: boolean; }, - catch: cause => new Error('Standalone large-output pipe returned invalid JSON.', {cause}), + catch: cause => new ScriptError('Standalone large-output pipe returned invalid JSON.', {cause}), }); }, ); @@ -263,17 +264,17 @@ const verifyMcp = Effect.fn('smokeSelfContained.verifyMcp')(function* ( yield* Effect.acquireUseRelease( Effect.tryPromise({ try: () => client.connect(transport), - catch: cause => new Error('Could not start the standalone MCP server.', {cause}), + catch: cause => new ScriptError('Could not start the standalone MCP server.', {cause}), }), () => Effect.tryPromise({ try: async () => { const tools = await client.listTools(); if (!tools.tools.some(tool => tool.name === 'recall_context')) { - throw new Error('Standalone MCP server did not expose recall_context.'); + throw new ScriptError('Standalone MCP server did not expose recall_context.'); } if (!tools.tools.some(tool => tool.name === 'inspect_code_graph')) { - throw new Error('Standalone MCP server did not expose inspect_code_graph.'); + throw new ScriptError('Standalone MCP server did not expose inspect_code_graph.'); } const recalled = await client.callTool( { @@ -289,7 +290,7 @@ const verifyMcp = Effect.fn('smokeSelfContained.verifyMcp')(function* ( ); const text = (recalled.content ?? []).map(item => ('text' in item ? item.text : '')).join('\n'); if (recalled.isError === true || !text.includes('standalone-bun-smoke.md')) { - throw new Error(`Standalone MCP recall missed the stored memory:\n${text}`); + throw new ScriptError(`Standalone MCP recall missed the stored memory:\n${text}`); } const graphOperations = [ {arguments: {callerCwd: cwd, operation: 'query', query: 'Greeter'}, expected: 'Greeter'}, @@ -316,16 +317,16 @@ const verifyMcp = Effect.fn('smokeSelfContained.verifyMcp')(function* ( ); const graphText = (inspected.content ?? []).map(item => ('text' in item ? item.text : '')).join('\n'); if (inspected.isError === true || !graphText.includes(graph.expected)) { - throw new Error(`Standalone MCP graph inspection missed ${graph.expected}:\n${graphText}`); + throw new ScriptError(`Standalone MCP graph inspection missed ${graph.expected}:\n${graphText}`); } } }, - catch: cause => (cause instanceof Error ? cause : new Error('Standalone MCP smoke failed.', {cause})), + catch: cause => scriptError(cause, 'Standalone MCP smoke failed.'), }), () => Effect.tryPromise({ try: () => client.close(), - catch: cause => new Error('Could not close the standalone MCP smoke client.', {cause}), + catch: cause => new ScriptError('Could not close the standalone MCP smoke client.', {cause}), }).pipe(Effect.catch(() => Effect.void)), ); }); @@ -334,4 +335,4 @@ const systemLayer = SystemInfo.layer; const commandLayer = CommandExecutor.layer.pipe(Layer.provide(systemLayer)); const smokeLayer = Layer.merge(systemLayer, commandLayer).pipe(Layer.provideMerge(BunServices.layer)); -BunRuntime.runMain(smokeSelfContained.pipe(Effect.provide(smokeLayer))); +BunRuntime.runMain(provideScriptLayer(smokeSelfContained, smokeLayer)); diff --git a/scripts/support/code-graph-workset-fixture.ts b/scripts/support/code-graph-workset-fixture.ts index 02e6518a..ca4edc98 100644 --- a/scripts/support/code-graph-workset-fixture.ts +++ b/scripts/support/code-graph-workset-fixture.ts @@ -1,3 +1,4 @@ +import {ScriptError} from '../effect/errors.js'; import {BunFileSystem, BunPath} from '@effect/platform-bun'; import {Effect, FileSystem, Layer, ManagedRuntime, Path} from 'effect'; import {sha256HexSync} from '../../src/crypto/sha256.js'; @@ -454,13 +455,13 @@ export function createCodeGraphWorksetFixturePlan( const stateProfile = options.stateProfile ?? 'all-clean'; const worksetName = options.worksetName ?? `code-graph-workset-${size}`; if (!/^[a-z0-9][a-z0-9._-]*$/.test(worksetName)) { - throw new Error(`Invalid code graph workset fixture name: ${worksetName}.`); + throw new ScriptError(`Invalid code graph workset fixture name: ${worksetName}.`); } const knownRepositoryKeys = new Set(Array.from({length: size}, (_, index) => repositoryKey(index))); for (const [key, state] of Object.entries(options.repositoryStates ?? {})) { if (!knownRepositoryKeys.has(key)) { - throw new Error(`Unknown code graph workset fixture repository: ${key}.`); + throw new ScriptError(`Unknown code graph workset fixture repository: ${key}.`); } assertFixtureState(state); } @@ -578,11 +579,11 @@ export async function materializeCodeGraphWorksetFixture( await mkdir(root, {recursive: true, mode: 0o700}); const existingEntries = await readdir(root); if (existingEntries.length > 0) { - throw new Error(`Code graph workset fixture root must be empty: ${root}.`); + throw new ScriptError(`Code graph workset fixture root must be empty: ${root}.`); } const concurrency = options.concurrency ?? 8; if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 32) { - throw new Error('Code graph workset fixture concurrency must be an integer from 1 through 32.'); + throw new ScriptError('Code graph workset fixture concurrency must be an integer from 1 through 32.'); } const home = join(root, 'home'); @@ -644,7 +645,7 @@ export async function establishCodeGraphWorksetStaleReadySnapshot( buildReadySnapshot: (repositoryPath: string) => Promise, ): Promise { if (repository.state !== 'stale' || !repository.readyCommit || !repository.headCommit) { - throw new Error(`Repository ${repository.repositoryKey} is not a materialized stale fixture member.`); + throw new ScriptError(`Repository ${repository.repositoryKey} is not a materialized stale fixture member.`); } await fixtureCheckout(repository.path, [ '-c', @@ -668,7 +669,7 @@ function archetypeForIndex(index: number): FixtureArchetype { : SCALE_ARCHETYPE_IDS[(index - CORE_ARCHETYPE_IDS.length) % SCALE_ARCHETYPE_IDS.length]; const archetype = id ? ARCHETYPE_BY_ID.get(id) : undefined; if (!archetype) { - throw new Error(`Missing code graph workset fixture archetype for repository ${index}.`); + throw new ScriptError(`Missing code graph workset fixture archetype for repository ${index}.`); } return archetype; } @@ -871,7 +872,7 @@ function fixtureQueries( const target = repositories[targetSize - 1]; if (!target) continue; if (target.archetype !== 'support') { - throw new Error(`Scale-tail fixture repository ${target.repositoryKey} must use the support archetype.`); + throw new ScriptError(`Scale-tail fixture repository ${target.repositoryKey} must use the support archetype.`); } const markerSymbol = repositoryMarkerSymbol(target.repositoryKey); queries.push( @@ -1048,7 +1049,7 @@ async function writeRepositoryFiles(root: string, files: readonly CodeGraphWorks await mkdir(root, {recursive: true}); for (const file of files) { if (file.path.startsWith('/') || file.path.split('/').includes('..')) { - throw new Error(`Unsafe code graph workset fixture path: ${file.path}.`); + throw new ScriptError(`Unsafe code graph workset fixture path: ${file.path}.`); } const target = join(root, file.path); await mkdir(dirname(target), {recursive: true}); @@ -1075,7 +1076,7 @@ function renderSeedManifest(plan: CodeGraphWorksetFixturePlan, repositoriesRoot: .map(key => { const project = projectByKey.get(key); if (!project) { - throw new Error(`Workset ${workset.name} references unknown fixture repository ${key}.`); + throw new ScriptError(`Workset ${workset.name} references unknown fixture repository ${key}.`); } return ` - ${project}\n`; }) @@ -1189,13 +1190,13 @@ function repositoryKey(index: number): string { function assertFixtureSize(size: number): asserts size is CodeGraphWorksetFixtureSize { if (!(CODE_GRAPH_WORKSET_FIXTURE_SUPPORTED_SIZES as readonly number[]).includes(size)) { - throw new Error(`Unsupported code graph workset fixture size: ${size}.`); + throw new ScriptError(`Unsupported code graph workset fixture size: ${size}.`); } } function assertFixtureState(state: string): asserts state is CodeGraphWorksetFixtureState { if (!['clean', 'cold', 'dirty', 'failed', 'missing', 'stale', 'worktree'].includes(state)) { - throw new Error(`Unsupported code graph workset fixture state: ${state}.`); + throw new ScriptError(`Unsupported code graph workset fixture state: ${state}.`); } } @@ -1229,7 +1230,7 @@ export async function readCodeGraphWorksetFixtureFile( ): Promise { const repository = fixture.repositories.find(candidate => candidate.repositoryKey === repositoryKey); if (!repository?.exists) { - throw new Error(`Code graph workset fixture repository is unavailable: ${repositoryKey}.`); + throw new ScriptError(`Code graph workset fixture repository is unavailable: ${repositoryKey}.`); } return readFile(join(repository.path, path), 'utf8'); } @@ -1280,10 +1281,10 @@ async function execFileAsync( maximumOutputBytes !== undefined && Buffer.byteLength(stderr, 'utf8') + Buffer.byteLength(stdout, 'utf8') > maximumOutputBytes ) { - throw new Error(`Command output exceeded ${maximumOutputBytes} bytes: ${executable}.`); + throw new ScriptError(`Command output exceeded ${maximumOutputBytes} bytes: ${executable}.`); } if (exitCode !== 0) { - throw new Error(stderr.trim() || `${executable} exited with status ${exitCode}.`); + throw new ScriptError(stderr.trim() || `${executable} exited with status ${exitCode}.`); } return {stderr, stdout}; } diff --git a/scripts/support/code-graph-workset-harness.ts b/scripts/support/code-graph-workset-harness.ts index 07866218..9f04a584 100644 --- a/scripts/support/code-graph-workset-harness.ts +++ b/scripts/support/code-graph-workset-harness.ts @@ -1,3 +1,4 @@ +import {scriptError, ScriptError} from '../effect/errors.js'; import {Clock, Effect} from 'effect'; import {CodeGraphIndexer, type CodeGraphIndexerShape} from '../../src/code_graph/indexer.js'; import { @@ -209,7 +210,7 @@ const publishIndexedCodeGraphWorksetCatalogScoped = Effect.fn('codeGraphWorksetH ]; }); if (members.length === 0) { - return yield* Effect.fail(new Error(`Fixture workset ${worksetName} has no ready routing projections.`)); + return yield* Effect.fail(new ScriptError(`Fixture workset ${worksetName} has no ready routing projections.`)); } const stagedGeneration = yield* stageCodeGraphWorksetCatalogGenerationFromReceipts(fixture.home, { manifestDigest: codeGraphWorksetManifestDigest(workset), @@ -427,7 +428,7 @@ export function codeGraphWorksetBenchmarkSample( ): CodeGraphWorksetBenchmarkSample { const measurement = measured.measurement; if (measurement.evidenceCardCount === 0 || measurement.timeToFirstEvidenceCardMilliseconds === undefined) { - throw new Error(`Workset benchmark control returned no evidence at size ${worksetSize}.`); + throw new ScriptError(`Workset benchmark control returned no evidence at size ${worksetSize}.`); } return { completionMilliseconds: measurement.completionMilliseconds, @@ -554,7 +555,7 @@ function indexFixtureRepository( if (repository.state === 'stale') { return Effect.tryPromise({ try: () => establishCodeGraphWorksetStaleReadySnapshot(repository, cwd => Effect.runPromise(index(cwd))), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => scriptError(cause), }).pipe(Effect.asVoid); } return index(repository.path).pipe( @@ -614,9 +615,9 @@ function fixtureRepositoryId(fixture: CodeGraphWorksetEvaluationFixtureV1, repos } function assertNonNegativeInteger(value: number, label: string): void { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${label} must be a non-negative safe integer.`); + if (!Number.isSafeInteger(value) || value < 0) throw new ScriptError(`${label} must be a non-negative safe integer.`); } function assertNonNegativeFinite(value: number, label: string): void { - if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be a non-negative finite number.`); + if (!Number.isFinite(value) || value < 0) throw new ScriptError(`${label} must be a non-negative finite number.`); } diff --git a/scripts/training/prepare-reviewed-recall-reranker-dataset.ts b/scripts/training/prepare-reviewed-recall-reranker-dataset.ts index 3c79e7f6..1469aee7 100644 --- a/scripts/training/prepare-reviewed-recall-reranker-dataset.ts +++ b/scripts/training/prepare-reviewed-recall-reranker-dataset.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from '../effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Layer, Path} from 'effect'; @@ -45,10 +46,10 @@ function parseArguments(args: readonly string[], resolve: (value: string) => str if (argument === '--draft') draft = resolve(required(args[++index], argument)); else if (argument === '--groups') groups = resolve(required(args[++index], argument)); else if (argument === '--output') output = resolve(required(args[++index], argument)); - else throw new Error(`Unknown reviewed-dataset preparation option: ${argument}\n\n${usage()}`); + else throw new ScriptError(`Unknown reviewed-dataset preparation option: ${argument}\n\n${usage()}`); } if (draft === undefined || groups === undefined || output === undefined) { - throw new Error(`--draft, --groups, and --output are required.\n\n${usage()}`); + throw new ScriptError(`--draft, --groups, and --output are required.\n\n${usage()}`); } return {draft, groups, output}; } @@ -57,12 +58,12 @@ function parseJson(content: string, source: string): unknown { try { return JSON.parse(content) as unknown; } catch (cause) { - throw new Error(`Could not parse JSON file: ${source}`, {cause}); + throw new ScriptError(`Could not parse JSON file: ${source}`, {cause}); } } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } @@ -80,4 +81,4 @@ function usage(): string { } const scriptLayer = Layer.mergeAll(BunServices.layer, SystemInfo.layer); -BunRuntime.runMain(program.pipe(Effect.provide(scriptLayer))); +BunRuntime.runMain(provideScriptLayer(program, scriptLayer)); diff --git a/scripts/training/recall-reranker-contract.ts b/scripts/training/recall-reranker-contract.ts index ab35fc5c..6e85f4f2 100644 --- a/scripts/training/recall-reranker-contract.ts +++ b/scripts/training/recall-reranker-contract.ts @@ -1,3 +1,4 @@ +import {ScriptError} from '../effect/errors.js'; import {Schema} from 'effect'; import {sha256HexSync} from '../../src/crypto/sha256.js'; import {detectSecretMatches} from '../../src/scrubber.js'; @@ -288,7 +289,7 @@ export function parseRecallRerankerValidationPolicyV1(value: unknown): RecallRer assertSha256(policy.forbiddenTextsSha256, 'validation policy forbidden texts'); assertSha256(policy.reservedEvaluation.sha256, 'validation policy reserved evaluation'); if (!isSafeRelativeFile(policy.receiptFile)) { - throw new Error('Recall reranker validation receipt must be a repository-relative file name.'); + throw new ScriptError('Recall reranker validation receipt must be a repository-relative file name.'); } return policy; } @@ -320,11 +321,11 @@ export function parseRecallRerankerGroupJsonLinesV1(content: string): readonly R try { value = JSON.parse(line) as unknown; } catch (cause) { - throw new Error(`Could not parse recall reranker JSONL line ${index + 1}.`, {cause}); + throw new ScriptError(`Could not parse recall reranker JSONL line ${index + 1}.`, {cause}); } groups.push(parseRecallRerankerQueryGroupV1(value)); } - if (groups.length === 0) throw new Error('Recall reranker dataset contains no query groups.'); + if (groups.length === 0) throw new ScriptError('Recall reranker dataset contains no query groups.'); return groups; } @@ -412,7 +413,7 @@ export function parseRecallRerankerDatasetV1( const manifest = parseRecallRerankerDatasetManifestV1(manifestValue); const groups = parseRecallRerankerGroupJsonLinesV1(groupContent); if (sha256HexSync(groupContent) !== manifest.groupFileSha256) { - throw new Error('Recall reranker group file checksum does not match its manifest.'); + throw new ScriptError('Recall reranker group file checksum does not match its manifest.'); } const dataset = {groups, manifest}; validateRecallRerankerDatasetV1(dataset, options); @@ -424,9 +425,9 @@ export function validateRecallRerankerDatasetV1( options: RecallRerankerValidationOptions = {}, ): void { const {groups, manifest} = dataset; - if (!manifest.privacyReviewed) throw new Error('Recall reranker dataset must pass privacy review.'); + if (!manifest.privacyReviewed) throw new ScriptError('Recall reranker dataset must pass privacy review.'); if (!isSafeRelativeFile(manifest.groupFile)) { - throw new Error('Recall reranker group file must be a repository-relative file name.'); + throw new ScriptError('Recall reranker group file must be a repository-relative file name.'); } assertSha256(manifest.groupFileSha256, 'group file'); assertSha256(manifest.groupsSha256, 'canonical groups'); @@ -435,11 +436,11 @@ export function validateRecallRerankerDatasetV1( const sourceIds = new Set(); for (const source of manifest.sources) { - if (sourceIds.has(source.id)) throw new Error(`Duplicate recall reranker source ID: ${source.id}`); + if (sourceIds.has(source.id)) throw new ScriptError(`Duplicate recall reranker source ID: ${source.id}`); sourceIds.add(source.id); validateSource(source); } - if (sourceIds.size === 0) throw new Error('Recall reranker dataset must declare at least one source.'); + if (sourceIds.size === 0) throw new ScriptError('Recall reranker dataset must declare at least one source.'); const groupIds = new Set(); const queryTexts = new Set(); @@ -449,7 +450,7 @@ export function validateRecallRerankerDatasetV1( const observedSplits = new Set(); for (const group of groups) { - if (groupIds.has(group.id)) throw new Error(`Duplicate recall reranker query group ID: ${group.id}`); + if (groupIds.has(group.id)) throw new ScriptError(`Duplicate recall reranker query group ID: ${group.id}`); groupIds.add(group.id); observedSplits.add(group.split); validateSafeText(group.id, `query group ${group.id} ID`); @@ -457,16 +458,17 @@ export function validateRecallRerankerDatasetV1( validateSafeText(group.provenanceRecord, `query group ${group.id} provenance record`); validateSafeText(group.query, `query group ${group.id} query`); if (!sourceIds.has(group.sourceId)) { - throw new Error(`Recall reranker query group ${group.id} references missing source: ${group.sourceId}`); + throw new ScriptError(`Recall reranker query group ${group.id} references missing source: ${group.sourceId}`); } const normalizedQuery = normalizeRecallRerankerText(group.query); - if (queryTexts.has(normalizedQuery)) throw new Error(`Duplicate normalized recall reranker query: ${group.id}`); + if (queryTexts.has(normalizedQuery)) + throw new ScriptError(`Duplicate normalized recall reranker query: ${group.id}`); queryTexts.add(normalizedQuery); assertNotReserved(normalizedQuery, forbiddenTexts, `query group ${group.id} query`); assertOneSplit(partitionSplits, normalizeRecallRerankerText(group.partitionKey), group.split, 'partition key'); if (group.candidates.length < 2 || group.candidates.length > RECALL_RERANKER_MAX_CANDIDATES) { - throw new Error( + throw new ScriptError( `Recall reranker query group ${group.id} must contain 2-${RECALL_RERANKER_MAX_CANDIDATES} candidates.`, ); } @@ -475,19 +477,21 @@ export function validateRecallRerankerDatasetV1( let negativeCount = 0; for (const candidate of group.candidates) { if (candidateIds.has(candidate.id)) { - throw new Error(`Duplicate recall reranker candidate ID ${candidate.id} in query group ${group.id}.`); + throw new ScriptError(`Duplicate recall reranker candidate ID ${candidate.id} in query group ${group.id}.`); } candidateIds.add(candidate.id); if (!sourceIds.has(candidate.sourceId)) { - throw new Error(`Recall reranker candidate ${candidate.id} references missing source: ${candidate.sourceId}`); + throw new ScriptError( + `Recall reranker candidate ${candidate.id} references missing source: ${candidate.sourceId}`, + ); } if (!Number.isInteger(candidate.relevance) || candidate.relevance < 0 || candidate.relevance > 3) { - throw new Error( + throw new ScriptError( `Recall reranker candidate ${candidate.id} has invalid relevance grade ${candidate.relevance}.`, ); } if (!candidate.reviewed) { - throw new Error(`Recall reranker candidate ${candidate.id} must be reviewed.`); + throw new ScriptError(`Recall reranker candidate ${candidate.id} must be reviewed.`); } validateSafeText(candidate.provenanceRecord, `candidate ${candidate.id} provenance record`); validateSafeText(candidate.text, `candidate ${candidate.id} text`); @@ -498,35 +502,37 @@ export function validateRecallRerankerDatasetV1( if (candidate.relevance > 0) { positiveCount += 1; if (candidate.negativeKind !== undefined) { - throw new Error(`Relevant recall reranker candidate ${candidate.id} cannot declare a negative kind.`); + throw new ScriptError(`Relevant recall reranker candidate ${candidate.id} cannot declare a negative kind.`); } } else { negativeCount += 1; if (candidate.negativeKind === undefined) { - throw new Error(`Negative recall reranker candidate ${candidate.id} must declare a negative kind.`); + throw new ScriptError(`Negative recall reranker candidate ${candidate.id} must declare a negative kind.`); } } } if (group.answerability === 'answerable' && (positiveCount === 0 || negativeCount === 0)) { - throw new Error(`Answerable recall reranker query group ${group.id} requires positive and negative candidates.`); + throw new ScriptError( + `Answerable recall reranker query group ${group.id} requires positive and negative candidates.`, + ); } if (group.answerability === 'no_answer' && positiveCount > 0) { - throw new Error(`No-answer recall reranker query group ${group.id} cannot contain relevant candidates.`); + throw new ScriptError(`No-answer recall reranker query group ${group.id} cannot contain relevant candidates.`); } } - if (groups.length === 0) throw new Error('Recall reranker dataset contains no query groups.'); + if (groups.length === 0) throw new ScriptError('Recall reranker dataset contains no query groups.'); if (options.requireAllSplits !== false) { for (const split of RECALL_RERANKER_SPLITS) { - if (!observedSplits.has(split)) throw new Error(`Recall reranker dataset is missing the ${split} split.`); + if (!observedSplits.has(split)) throw new ScriptError(`Recall reranker dataset is missing the ${split} split.`); } } const counts = recallRerankerDatasetCountsV1(groups); if (JSON.stringify(counts) !== JSON.stringify(manifest.counts)) { - throw new Error('Recall reranker dataset counts do not match its manifest.'); + throw new ScriptError('Recall reranker dataset counts do not match its manifest.'); } if (recallRerankerGroupsHashV1(groups) !== manifest.groupsSha256) { - throw new Error('Recall reranker canonical group hash does not match its manifest.'); + throw new ScriptError('Recall reranker canonical group hash does not match its manifest.'); } } @@ -581,38 +587,39 @@ function validateSource(source: RecallRerankerSourceV1): void { ] as const) { validateSafeText(value, `source ${source.id} ${label}`); } - if (!source.trainingApproved) throw new Error(`Recall reranker source ${source.id} is not approved for training.`); + if (!source.trainingApproved) + throw new ScriptError(`Recall reranker source ${source.id} is not approved for training.`); if (!source.redistributionApproved) { - throw new Error(`Recall reranker source ${source.id} is not approved for redistribution.`); + throw new ScriptError(`Recall reranker source ${source.id} is not approved for redistribution.`); } if (source.kind === 'self_authored_synthetic' && source.privacyBasis !== 'self_authored') { - throw new Error(`Self-authored source ${source.id} must use the self_authored privacy basis.`); + throw new ScriptError(`Self-authored source ${source.id} must use the self_authored privacy basis.`); } if ( (source.kind === 'public_dataset' || source.kind === 'public_repository') && source.privacyBasis !== 'public_licensed' ) { - throw new Error(`Public source ${source.id} must use the public_licensed privacy basis.`); + throw new ScriptError(`Public source ${source.id} must use the public_licensed privacy basis.`); } if (source.kind === 'opt_in_sanitized') { if (source.privacyBasis !== 'explicit_opt_in' || !source.consentReference?.trim()) { - throw new Error(`Opt-in source ${source.id} requires an explicit consent reference.`); + throw new ScriptError(`Opt-in source ${source.id} requires an explicit consent reference.`); } validateSafeText(source.consentReference, `source ${source.id} consent reference`); } else if (source.consentReference !== undefined) { - throw new Error(`Non-opt-in source ${source.id} cannot declare a consent reference.`); + throw new ScriptError(`Non-opt-in source ${source.id} cannot declare a consent reference.`); } } function validateSafeText(value: string, label: string): void { const matches = detectSecretMatches(value); - if (matches.length > 0) throw new Error(`${label} contains sensitive data (${matches.join(', ')}).`); + if (matches.length > 0) throw new ScriptError(`${label} contains sensitive data (${matches.join(', ')}).`); if ( /(?:^|[\s"'`(])(?:[A-Za-z]:[\\/](?:Users|Documents and Settings)[\\/]|\/mnt\/[a-z]\/(?:Users|home)\/|\\\\[^\\\s]+\\[^\\\s]+)/i.test( value, ) ) { - throw new Error(`${label} contains an absolute local path.`); + throw new ScriptError(`${label} contains an absolute local path.`); } } @@ -624,17 +631,17 @@ function assertOneSplit( ): void { const previous = seen.get(key); if (previous !== undefined && previous !== split) { - throw new Error(`Recall reranker ${label} leaks across ${previous} and ${split} splits.`); + throw new ScriptError(`Recall reranker ${label} leaks across ${previous} and ${split} splits.`); } seen.set(key, split); } function assertNotReserved(value: string, forbidden: ReadonlySet, label: string): void { - if (forbidden.has(value)) throw new Error(`${label} duplicates reserved evaluation content.`); + if (forbidden.has(value)) throw new ScriptError(`${label} duplicates reserved evaluation content.`); } function assertSha256(value: string, label: string): void { - if (!/^[0-9a-f]{64}$/.test(value)) throw new Error(`Recall reranker ${label} SHA-256 is invalid.`); + if (!/^[0-9a-f]{64}$/.test(value)) throw new ScriptError(`Recall reranker ${label} SHA-256 is invalid.`); } function isSafeRelativeFile(value: string): boolean { diff --git a/scripts/training/recall-reranker-parity.ts b/scripts/training/recall-reranker-parity.ts index c2d85202..04f9f9b0 100644 --- a/scripts/training/recall-reranker-parity.ts +++ b/scripts/training/recall-reranker-parity.ts @@ -1,3 +1,4 @@ +import {ScriptError} from '../effect/errors.js'; import {Schema} from 'effect'; export const RECALL_RERANKER_PARITY_FIXTURE_VERSION = 1 as const; @@ -150,29 +151,29 @@ export function parseRecallRerankerParityFixtureV1(value: unknown): RecallRerank fixture.run.runJsonSha256, ]; if (shaValues.some(value => !/^[0-9a-f]{64}$/.test(value))) { - throw new Error('Recall reranker parity fixture contains an invalid SHA-256.'); + throw new ScriptError('Recall reranker parity fixture contains an invalid SHA-256.'); } if (!/^[0-9a-f]{40}$/.test(fixture.run.trainingCodeRevision)) { - throw new Error('Recall reranker parity fixture must pin an immutable training source revision.'); + throw new ScriptError('Recall reranker parity fixture must pin an immutable training source revision.'); } if ( fixture.groups.length === 0 || fixture.selection.maximumGroups <= 0 || fixture.groups.length > fixture.selection.maximumGroups ) { - throw new Error('Recall reranker parity fixture contains an invalid validation-group selection.'); + throw new ScriptError('Recall reranker parity fixture contains an invalid validation-group selection.'); } if (fixture.runtimeTarget.contextLimit <= 0 || fixture.runtimeTarget.documentCharacterLimit <= 0) { - throw new Error('Recall reranker parity fixture contains invalid runtime limits.'); + throw new ScriptError('Recall reranker parity fixture contains invalid runtime limits.'); } const groupIds = new Set(); for (const group of fixture.groups) { if (!group.groupId.trim() || !group.query.trim() || groupIds.has(group.groupId)) { - throw new Error('Recall reranker parity fixture contains an invalid or duplicate group.'); + throw new ScriptError('Recall reranker parity fixture contains an invalid or duplicate group.'); } groupIds.add(group.groupId); if (group.candidates.length < 2) { - throw new Error(`Recall reranker parity group ${group.groupId} requires at least two candidates.`); + throw new ScriptError(`Recall reranker parity group ${group.groupId} requires at least two candidates.`); } const candidateIds = new Set(); for (const candidate of group.candidates) { @@ -185,7 +186,7 @@ export function parseRecallRerankerParityFixtureV1(value: unknown): RecallRerank candidate.relevance < 0 || candidate.relevance > 3 ) { - throw new Error(`Recall reranker parity group ${group.groupId} contains an invalid candidate.`); + throw new ScriptError(`Recall reranker parity group ${group.groupId} contains an invalid candidate.`); } candidateIds.add(candidate.candidateId); } @@ -210,7 +211,7 @@ export function evaluateRecallRerankerParity( for (const group of fixture.groups) { const scores = nativeScores.get(group.groupId); if (!scores || scores.length !== group.candidates.length || scores.some(score => !Number.isFinite(score))) { - throw new Error(`Native reranker returned invalid scores for parity group ${group.groupId}.`); + throw new ScriptError(`Native reranker returned invalid scores for parity group ${group.groupId}.`); } const candidates = group.candidates.map((candidate, index) => { const nativeScore = scores[index]!; @@ -246,7 +247,7 @@ export function evaluateRecallRerankerParity( groups.push({candidates, groupId: group.groupId}); } if (nativeScores.size !== fixture.groups.length) { - throw new Error('Native reranker parity scores contain unexpected groups.'); + throw new ScriptError('Native reranker parity scores contain unexpected groups.'); } return { absoluteError: { @@ -268,6 +269,6 @@ function validateThresholds(thresholds: RecallRerankerParityThresholds): void { !Number.isFinite(thresholds.minimumOrderingGap) || thresholds.minimumOrderingGap < 0 ) { - throw new Error('Recall reranker parity thresholds must be finite non-negative numbers.'); + throw new ScriptError('Recall reranker parity thresholds must be finite non-negative numbers.'); } } diff --git a/scripts/training/recall-reranker-preparation.ts b/scripts/training/recall-reranker-preparation.ts index bd4777c3..5df4eb6a 100644 --- a/scripts/training/recall-reranker-preparation.ts +++ b/scripts/training/recall-reranker-preparation.ts @@ -1,3 +1,4 @@ +import {ScriptError} from '../effect/errors.js'; import {sha256HexSync} from '../../src/crypto/sha256.js'; import { createRecallEvaluationFixtureV2, @@ -16,13 +17,13 @@ export function prepareReviewedRecallRerankerDatasetV1( ): RecallRerankerDatasetV1 { const draft = parseRecallRerankerDatasetDraftV1(draftValue); if (draft.purpose !== 'training_candidate') { - throw new Error('Reviewed dataset preparation only accepts purpose training_candidate.'); + throw new ScriptError('Reviewed dataset preparation only accepts purpose training_candidate.'); } if ((draft.reservedEvaluations?.length ?? 0) > 0) { - throw new Error('The preparation helper manages reserved evaluations; remove them from the draft.'); + throw new ScriptError('The preparation helper manages reserved evaluations; remove them from the draft.'); } if (draft.groupFile !== undefined && draft.groupFile !== 'groups.jsonl') { - throw new Error('The preparation helper writes the reviewed groups to groups.jsonl.'); + throw new ScriptError('The preparation helper writes the reviewed groups to groups.jsonl.'); } const fixture = createRecallEvaluationFixtureV2(); diff --git a/scripts/validate-mixed-nx-bazel-gate.ts b/scripts/validate-mixed-nx-bazel-gate.ts index fbd3358a..814e399a 100644 --- a/scripts/validate-mixed-nx-bazel-gate.ts +++ b/scripts/validate-mixed-nx-bazel-gate.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import {BunRuntime} from '@effect/platform-bun'; import {Effect, FileSystem, Path} from 'effect'; import {ApplicationLayer} from '../src/effect/runtime.js'; @@ -48,7 +49,7 @@ export function validateMixedNxBazelGate(evidence: MixedNxBazelGateEvidence, bud ) { failures.push('post-target closure was not bounded'); } - if (failures.length > 0) throw new Error(`Mixed Nx/Bazel P2 gate failed: ${failures.join('; ')}`); + if (failures.length > 0) throw new ScriptError(`Mixed Nx/Bazel P2 gate failed: ${failures.join('; ')}`); } const run = Effect.fn('mixedNxBazelGate.run')(function* (args: readonly string[] = process.argv.slice(2)) { @@ -57,9 +58,9 @@ const run = Effect.fn('mixedNxBazelGate.run')(function* (args: readonly string[] for (let index = 0; index < args.length; index += 1) { if (args[index] === '--evidence') evidencePath = args[++index]; else if (args[index] === '--budgets') budgetsPath = args[++index]!; - else throw new Error(`Unknown mixed-monorepo gate option: ${args[index]}`); + else throw new ScriptError(`Unknown mixed-monorepo gate option: ${args[index]}`); } - if (!evidencePath) throw new Error('--evidence requires a JSON artifact.'); + if (!evidencePath) throw new ScriptError('--evidence requires a JSON artifact.'); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const [evidence, budgets] = yield* Effect.all([ @@ -69,4 +70,4 @@ const run = Effect.fn('mixedNxBazelGate.run')(function* (args: readonly string[] validateMixedNxBazelGate(evidence as MixedNxBazelGateEvidence, budgets as MixedNxBazelGateBudgets); }); -if (import.meta.main) BunRuntime.runMain(run().pipe(Effect.provide(ApplicationLayer))); +if (import.meta.main) BunRuntime.runMain(provideScriptLayer(run(), ApplicationLayer)); diff --git a/scripts/validate-recall-reranker-dataset.ts b/scripts/validate-recall-reranker-dataset.ts index 2660c9ee..a603bea3 100644 --- a/scripts/validate-recall-reranker-dataset.ts +++ b/scripts/validate-recall-reranker-dataset.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Console, Effect, FileSystem, Layer, Path} from 'effect'; @@ -50,7 +51,7 @@ const program = Effect.gen(function* () { policy.forbiddenTextsSha256 !== forbiddenTextsHash ) { return yield* Effect.fail( - new Error('Recall reranker validation policy does not match the current frozen recall evaluation fixture.'), + new ScriptError('Recall reranker validation policy does not match the current frozen recall evaluation fixture.'), ); } if ( @@ -58,7 +59,9 @@ const program = Effect.gen(function* () { candidate => candidate.name === policy.reservedEvaluation.name && candidate.sha256 === evaluationHash, ) ) { - return yield* Effect.fail(new Error('Dataset manifest does not reserve the current recall evaluation fixture.')); + return yield* Effect.fail( + new ScriptError('Dataset manifest does not reserve the current recall evaluation fixture.'), + ); } const dataset = parseRecallRerankerDatasetV1(manifestValue, groupContent, { forbiddenTexts, @@ -77,7 +80,7 @@ function parseJson(content: string, source: string): unknown { try { return JSON.parse(content) as unknown; } catch (cause) { - throw new Error(`Could not parse JSON file: ${source}`, {cause}); + throw new ScriptError(`Could not parse JSON file: ${source}`, {cause}); } } @@ -90,15 +93,15 @@ function parseArguments(args: readonly string[], resolve: (value: string) => str for (let index = 0; index < args.length; index += 1) { const argument = args[index]!; if (argument === '--dataset') dataset = resolve(required(args[++index], argument)); - else throw new Error(`Unknown recall reranker validation option: ${argument}. Pass --help for usage.`); + else throw new ScriptError(`Unknown recall reranker validation option: ${argument}. Pass --help for usage.`); } return {dataset}; } function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } const scriptLayer = Layer.mergeAll(BunServices.layer, SystemInfo.layer); -BunRuntime.runMain(program.pipe(Effect.provide(scriptLayer))); +BunRuntime.runMain(provideScriptLayer(program, scriptLayer)); diff --git a/scripts/validate-recall-reranker-parity.ts b/scripts/validate-recall-reranker-parity.ts index 64fde4de..3db857bb 100644 --- a/scripts/validate-recall-reranker-parity.ts +++ b/scripts/validate-recall-reranker-parity.ts @@ -1,3 +1,4 @@ +import {provideScriptLayer, ScriptError} from './effect/errors.js'; import * as BunRuntime from '@effect/platform-bun/BunRuntime'; import * as BunServices from '@effect/platform-bun/BunServices'; import {Clock, Effect, FileSystem, Layer, Path} from 'effect'; @@ -109,7 +110,7 @@ function parseArguments(args: readonly string[], resolve: (value: string) => str minimumOrderingGap = numberValue(args[++index], argument); } else if (argument === '--model') model = resolve(required(args[++index], argument)); else if (argument === '--output') output = resolve(required(args[++index], argument)); - else throw new Error(`Unknown reranker-parity option: ${argument}`); + else throw new ScriptError(`Unknown reranker-parity option: ${argument}`); } return { fixture: required(fixture, '--fixture'), @@ -129,13 +130,13 @@ function verifyRuntimeBinding( }, manifest: LocalModelManifest, ): void { - if (manifest.role !== 'reranker') throw new Error(`Local model ${manifest.id} is not a reranker.`); + if (manifest.role !== 'reranker') throw new ScriptError(`Local model ${manifest.id} is not a reranker.`); if ( manifest.architecture !== expected.architecture || manifest.contextLimit !== expected.contextLimit || manifest.runtime.nodeLlamaCpp !== expected.nodeLlamaCpp ) { - throw new Error('Local reranker manifest does not match the Python parity fixture runtime target.'); + throw new ScriptError('Local reranker manifest does not match the Python parity fixture runtime target.'); } } @@ -145,24 +146,24 @@ const verifyModelArtifact = Effect.fn('validateRecallRerankerParity.verifyModelA ) { const fs = yield* FileSystem.FileSystem; const info = yield* fs.stat(modelPath); - if (info.type !== 'File') throw new Error(`Local reranker artifact is not a regular file: ${modelPath}`); + if (info.type !== 'File') throw new ScriptError(`Local reranker artifact is not a regular file: ${modelPath}`); if (Number(info.size) !== manifest.size) { - throw new Error(`Local reranker size ${info.size} does not match manifest size ${manifest.size}.`); + throw new ScriptError(`Local reranker size ${info.size} does not match manifest size ${manifest.size}.`); } const digest = yield* sha256FileHex(modelPath); if (digest !== manifest.sha256) { - throw new Error(`Local reranker SHA-256 ${digest} does not match manifest SHA-256 ${manifest.sha256}.`); + throw new ScriptError(`Local reranker SHA-256 ${digest} does not match manifest SHA-256 ${manifest.sha256}.`); } }); function required(value: string | undefined, option: string): string { - if (!value?.trim()) throw new Error(`${option} requires a value.`); + if (!value?.trim()) throw new ScriptError(`${option} requires a value.`); return value; } function numberValue(value: string | undefined, option: string): number { const parsed = Number(required(value, option)); - if (!Number.isFinite(parsed) || parsed < 0) throw new Error(`${option} requires a finite non-negative number.`); + if (!Number.isFinite(parsed) || parsed < 0) throw new ScriptError(`${option} requires a finite non-negative number.`); return parsed; } @@ -170,4 +171,4 @@ const systemLayer = SystemInfo.layer; const runtimeLayer = isolatedLocalModelRuntimeLayer().pipe(Layer.provideMerge(systemLayer)); const ParityLayer = Layer.mergeAll(runtimeLayer, systemLayer).pipe(Layer.provideMerge(BunServices.layer)); -BunRuntime.runMain(validateParity.pipe(Effect.provide(ParityLayer))); +BunRuntime.runMain(provideScriptLayer(validateParity, ParityLayer)); diff --git a/src/candidate_memory.ts b/src/candidate_memory.ts index 1bb8a774..112376c3 100644 --- a/src/candidate_memory.ts +++ b/src/candidate_memory.ts @@ -89,6 +89,10 @@ interface CandidateAuditTransition { readonly memoryUri?: string; } +class CandidateMemoryError extends Error { + readonly _tag = 'CandidateMemoryError' as const; +} + const MEMORY_READ_CONCURRENCY = 16; const MAX_CANDIDATE_AUDIT_EVENTS = 5_000; const MAX_CANDIDATE_REVIEW_AUDIT_EVENTS = 100; @@ -353,7 +357,7 @@ export const saveCandidateReview = Effect.fn('candidate.saveReview')(function* ( const serialized = `${JSON.stringify(review, undefined, 2)}\n`; if (new TextEncoder().encode(serialized).byteLength > MAX_CANDIDATE_REVIEW_BYTES) { return yield* Effect.fail( - new Error(`Candidate review exceeds the ${MAX_CANDIDATE_REVIEW_BYTES}-byte persistence limit.`), + new CandidateMemoryError(`Candidate review exceeds the ${MAX_CANDIDATE_REVIEW_BYTES}-byte persistence limit.`), ); } yield* writePrivateFileAtomically(fs, path, serialized); @@ -372,7 +376,7 @@ export const loadCandidateReview = Effect.fn('candidate.loadReview')(function* ( const raw = yield* fs.readFileString(path); const review = yield* Effect.try({ try: () => parseCandidateReview(JSON.parse(raw)), - catch: cause => new Error(`Invalid candidate review ${reviewId}: ${errorText(cause)}`), + catch: cause => new CandidateMemoryError(`Invalid candidate review ${reviewId}: ${errorText(cause)}`), }); yield* syncCandidateAudit(agentContextHome, review.auditEvents); return review; @@ -760,7 +764,7 @@ function parseCandidateReview(value: unknown): CandidateReview { !('candidates' in value) || !Array.isArray(value.candidates) ) { - throw new Error('unsupported review document'); + throw new CandidateMemoryError('unsupported review document'); } const review = value as CandidateReview; return { diff --git a/src/cli_ui.ts b/src/cli_ui.ts index 97c39ed7..4152f0a8 100644 --- a/src/cli_ui.ts +++ b/src/cli_ui.ts @@ -167,7 +167,7 @@ const startSpinner = Effect.fn('cliUi.startSpinner')(function* (message: string) export interface ProgressIndicator { update(message: string): Effect.Effect; - stop(): Effect.Effect; + readonly stop: Effect.Effect; } interface LineProgressState { @@ -227,19 +227,18 @@ export const startProgress = Effect.fn('cliUi.startProgress')(function* (message yield* Ref.set(state, {...current, pendingMessage: nextMessage}); }), ), - stop: () => - gate.withPermit( - Effect.gen(function* () { - const current = yield* Ref.get(state); - if (current.pendingMessage === undefined || current.pendingMessage === current.lastEmittedMessage) return; - yield* Console.log(current.pendingMessage); - yield* Ref.set(state, { - family: progressMessageFamily(current.pendingMessage), - lastEmittedAtMilliseconds: yield* Clock.currentTimeMillis, - lastEmittedMessage: current.pendingMessage, - }); - }), - ), + stop: gate.withPermit( + Effect.gen(function* () { + const current = yield* Ref.get(state); + if (current.pendingMessage === undefined || current.pendingMessage === current.lastEmittedMessage) return; + yield* Console.log(current.pendingMessage); + yield* Ref.set(state, { + family: progressMessageFamily(current.pendingMessage), + lastEmittedAtMilliseconds: yield* Clock.currentTimeMillis, + lastEmittedMessage: current.pendingMessage, + }); + }), + ), }; } @@ -261,7 +260,7 @@ export const startProgress = Effect.fn('cliUi.startProgress')(function* (message const fiber = yield* render.pipe(Effect.repeat(Schedule.spaced(100)), Effect.forkDetach); return { update: (nextMessage: string) => Ref.set(currentMessage, nextMessage).pipe(Effect.andThen(render)), - stop: () => Fiber.interrupt(fiber).pipe(Effect.andThen(flush), Effect.andThen(terminal.display('\r\u001b[2K'))), + stop: Fiber.interrupt(fiber).pipe(Effect.andThen(flush), Effect.andThen(terminal.display('\r\u001b[2K'))), }; }); diff --git a/src/code_graph/analysis.ts b/src/code_graph/analysis.ts index 8e98bc1c..5360ae59 100644 --- a/src/code_graph/analysis.ts +++ b/src/code_graph/analysis.ts @@ -1,13 +1,15 @@ import {Clock, Context, Effect, Layer, Option} from 'effect'; import {sha256HexSync} from '../crypto/sha256.js'; +import { + positiveInteger, + resolveBudget, + resolveLimits, + type CodeGraphAnalysisOptions, + type ResolvedCodeGraphAnalysisBudget, + type ResolvedCodeGraphAnalysisLimits, +} from './analysis_configuration.js'; import {compareCodeUnits} from './ordering.js'; -import type { - CodeGraphEdge, - CodeGraphProvenance, - CodeGraphRelation, - CodeGraphSnapshot, - CodeGraphSymbol, -} from './types.js'; +import type {CodeGraphEdge, CodeGraphProvenance, CodeGraphRelation, CodeGraphSymbol} from './types.js'; import type { CodeGraphAnalysisEdgeAggregate, CodeGraphAnalysisEdgeAggregatePage, @@ -21,62 +23,13 @@ import {CodeGraphStore} from './store.js'; export const CODE_GRAPH_ANALYSIS_VERSION = 3 as const; -export interface CodeGraphAnalysisBudget { - /** Rows grouped by each interruptible aggregate query. */ - readonly aggregatePageSize?: number; - /** Maximum distinct edge rows considered by the topology pass. */ - readonly maxEdges?: number; - /** Maximum edge row visits across the topology and scoring passes. */ - readonly maxEdgeVisits?: number; - readonly maxDurationMilliseconds?: number; - readonly maxNodes?: number; - readonly pageSize?: number; -} - -export interface CodeGraphAnalysisLimits { - readonly communities?: number; - readonly communityMembers?: number; - readonly components?: number; - readonly confidenceFindings?: number; - readonly hubs?: number; - readonly memberships?: number; - readonly relationshipGroupMembers?: number; - readonly relationshipGroups?: number; - readonly surprisingLinks?: number; -} - -export interface CodeGraphAnalysisOptions { - readonly allowedProvenances?: readonly CodeGraphProvenance[]; - readonly budget?: CodeGraphAnalysisBudget; - /** Stable `cgc_…` identifier returned by an earlier analysis. */ - readonly communityId?: string; - readonly databasePath: string; - readonly limits?: CodeGraphAnalysisLimits; - readonly minimumGodNodeDegree?: number; - readonly minimumHubDegree?: number; - readonly snapshot: CodeGraphSnapshot; -} - -export interface ResolvedCodeGraphAnalysisBudget { - readonly aggregatePageSize: number; - readonly maxEdges: number; - readonly maxEdgeVisits: number; - readonly maxDurationMilliseconds: number; - readonly maxNodes: number; - readonly pageSize: number; -} - -export interface ResolvedCodeGraphAnalysisLimits { - readonly communities: number; - readonly communityMembers: number; - readonly components: number; - readonly confidenceFindings: number; - readonly hubs: number; - readonly memberships: number; - readonly relationshipGroupMembers: number; - readonly relationshipGroups: number; - readonly surprisingLinks: number; -} +export type { + CodeGraphAnalysisBudget, + CodeGraphAnalysisLimits, + CodeGraphAnalysisOptions, + ResolvedCodeGraphAnalysisBudget, + ResolvedCodeGraphAnalysisLimits, +} from './analysis_configuration.js'; export interface CodeGraphAnalysisCount { readonly count: number; @@ -368,24 +321,6 @@ export class CodeGraphAnalysis extends Context.Service ALL_PROVENANCES.has(value)))].sort(); } - -function positiveInteger(value: number | undefined, fallback: number, minimum: number, maximum: number): number { - return Number.isSafeInteger(value) ? Math.max(minimum, Math.min(maximum, value!)) : fallback; -} - -function nonNegativeInteger(value: number | undefined, fallback: number, maximum: number): number { - return Number.isSafeInteger(value) ? Math.max(0, Math.min(maximum, value!)) : fallback; -} - -function nonNegativeSafeInteger(value: number | undefined, fallback: number): number { - const candidate = Number.isSafeInteger(value) && value! >= 0 ? value! : fallback; - return Number.isSafeInteger(candidate) && candidate >= 0 ? candidate : 0; -} - -function saturatingMultiply(value: number, multiplier: number): number { - return Math.min(Number.MAX_SAFE_INTEGER, value * multiplier); -} diff --git a/src/code_graph/analysis_configuration.ts b/src/code_graph/analysis_configuration.ts new file mode 100644 index 00000000..642f4c13 --- /dev/null +++ b/src/code_graph/analysis_configuration.ts @@ -0,0 +1,131 @@ +import type {CodeGraphProvenance, CodeGraphSnapshot} from './types.js'; + +export interface CodeGraphAnalysisBudget { + /** Rows grouped by each interruptible aggregate query. */ + readonly aggregatePageSize?: number; + /** Maximum distinct edge rows considered by the topology pass. */ + readonly maxEdges?: number; + /** Maximum edge row visits across the topology and scoring passes. */ + readonly maxEdgeVisits?: number; + readonly maxDurationMilliseconds?: number; + readonly maxNodes?: number; + readonly pageSize?: number; +} + +export interface CodeGraphAnalysisLimits { + readonly communities?: number; + readonly communityMembers?: number; + readonly components?: number; + readonly confidenceFindings?: number; + readonly hubs?: number; + readonly memberships?: number; + readonly relationshipGroupMembers?: number; + readonly relationshipGroups?: number; + readonly surprisingLinks?: number; +} + +export interface CodeGraphAnalysisOptions { + readonly allowedProvenances?: readonly CodeGraphProvenance[]; + readonly budget?: CodeGraphAnalysisBudget; + /** Stable `cgc_…` identifier returned by an earlier analysis. */ + readonly communityId?: string; + readonly databasePath: string; + readonly limits?: CodeGraphAnalysisLimits; + readonly minimumGodNodeDegree?: number; + readonly minimumHubDegree?: number; + readonly snapshot: CodeGraphSnapshot; +} + +export interface ResolvedCodeGraphAnalysisBudget { + readonly aggregatePageSize: number; + readonly maxEdges: number; + readonly maxEdgeVisits: number; + readonly maxDurationMilliseconds: number; + readonly maxNodes: number; + readonly pageSize: number; +} + +export interface ResolvedCodeGraphAnalysisLimits { + readonly communities: number; + readonly communityMembers: number; + readonly components: number; + readonly confidenceFindings: number; + readonly hubs: number; + readonly memberships: number; + readonly relationshipGroupMembers: number; + readonly relationshipGroups: number; + readonly surprisingLinks: number; +} + +const DEFAULT_BUDGET = { + aggregatePageSize: 50_000, + maxDurationMilliseconds: 60_000, + pageSize: 1_000, +} as const; + +const DEFAULT_LIMITS: ResolvedCodeGraphAnalysisLimits = { + communities: 250, + communityMembers: 100, + components: 250, + confidenceFindings: 50, + hubs: 50, + memberships: 25_000, + relationshipGroupMembers: 20, + relationshipGroups: 50, + surprisingLinks: 50, +}; + +export function resolveBudget( + input: CodeGraphAnalysisBudget | undefined, + snapshot: CodeGraphSnapshot, +): ResolvedCodeGraphAnalysisBudget { + const maxEdges = nonNegativeSafeInteger(input?.maxEdges, snapshot.edgeCount); + return { + aggregatePageSize: positiveInteger(input?.aggregatePageSize, DEFAULT_BUDGET.aggregatePageSize, 1, 250_000), + maxEdges, + maxEdgeVisits: nonNegativeSafeInteger(input?.maxEdgeVisits, saturatingMultiply(maxEdges, 2)), + maxDurationMilliseconds: positiveInteger( + input?.maxDurationMilliseconds, + DEFAULT_BUDGET.maxDurationMilliseconds, + 1, + 10 * 60_000, + ), + maxNodes: nonNegativeSafeInteger(input?.maxNodes, snapshot.symbolCount), + pageSize: positiveInteger(input?.pageSize, DEFAULT_BUDGET.pageSize, 1, 2_000), + }; +} + +export function resolveLimits(input: CodeGraphAnalysisLimits | undefined): ResolvedCodeGraphAnalysisLimits { + return { + communities: nonNegativeInteger(input?.communities, DEFAULT_LIMITS.communities, 5_000), + communityMembers: nonNegativeInteger(input?.communityMembers, DEFAULT_LIMITS.communityMembers, 5_000), + components: nonNegativeInteger(input?.components, DEFAULT_LIMITS.components, 5_000), + confidenceFindings: nonNegativeInteger(input?.confidenceFindings, DEFAULT_LIMITS.confidenceFindings, 500), + hubs: nonNegativeInteger(input?.hubs, DEFAULT_LIMITS.hubs, 500), + memberships: nonNegativeInteger(input?.memberships, DEFAULT_LIMITS.memberships, 250_000), + relationshipGroupMembers: nonNegativeInteger( + input?.relationshipGroupMembers, + DEFAULT_LIMITS.relationshipGroupMembers, + 500, + ), + relationshipGroups: nonNegativeInteger(input?.relationshipGroups, DEFAULT_LIMITS.relationshipGroups, 500), + surprisingLinks: nonNegativeInteger(input?.surprisingLinks, DEFAULT_LIMITS.surprisingLinks, 500), + }; +} + +export function positiveInteger(value: number | undefined, fallback: number, minimum: number, maximum: number): number { + return Number.isSafeInteger(value) ? Math.max(minimum, Math.min(maximum, value!)) : fallback; +} + +function nonNegativeInteger(value: number | undefined, fallback: number, maximum: number): number { + return Number.isSafeInteger(value) ? Math.max(0, Math.min(maximum, value!)) : fallback; +} + +function nonNegativeSafeInteger(value: number | undefined, fallback: number): number { + const candidate = Number.isSafeInteger(value) && value! >= 0 ? value! : fallback; + return Number.isSafeInteger(candidate) && candidate >= 0 ? candidate : 0; +} + +function saturatingMultiply(value: number, multiplier: number): number { + return Math.min(Number.MAX_SAFE_INTEGER, value * multiplier); +} diff --git a/src/code_graph/automatic_compaction.ts b/src/code_graph/automatic_compaction.ts new file mode 100644 index 00000000..e00cdeac --- /dev/null +++ b/src/code_graph/automatic_compaction.ts @@ -0,0 +1,683 @@ +import {Clock, Crypto, Effect, FileSystem, Option, Path, Stdio, Stream} from 'effect'; +import {CommandExecutor, type CommandExecutionError} from '../effect/command.js'; +import {runtimeTextDirectoryNamePage, SystemInfo, type SystemInfoShape} from '../effect/system.js'; +import {CODE_GRAPH_COMPACTION_WORKER_ARGUMENT} from '../worker_protocol.js'; +import { + CODE_GRAPH_AUTOMATIC_COMPACTION_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_DEFERRED_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_FAILURE_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_BYTES, + CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_COOLDOWN_MILLISECONDS, + claimCodeGraphAutomaticCompactionCandidate, + codeGraphAutomaticCompactionCandidateAllowed, + codeGraphAutomaticCompactionCooldownMilliseconds, + recordCodeGraphAutomaticCompactionAttempt, +} from './automatic_compaction_receipt.js'; +import {codeGraphRepositoriesRoot} from './layout.js'; +import {compareCodeUnits} from './ordering.js'; +import {CODE_GRAPH_SCHEMA_VERSION} from './types.js'; +import { + codeGraphCompactionRequiredFreeBytes, + compactCodeGraphStorage, + inspectCodeGraphStorage, + type CodeGraphActiveStorage, + type CodeGraphCompactionSummary, +} from './storage.js'; + +class CodeGraphAutomaticCompactionError extends Error { + readonly _tag = 'CodeGraphAutomaticCompactionError' as const; +} + +export const CODE_GRAPH_AUTOMATIC_COMPACTION_INITIAL_DELAY_MILLISECONDS = 15_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_INTERVAL_MILLISECONDS = 60_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_DATABASE_LIMIT = 128; +const CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL = 1; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_INPUT_BYTES_MAXIMUM = 16 * 1_024; +const CODE_GRAPH_AUTOMATIC_COMPACTION_OUTPUT_BYTES_MAXIMUM = 4 * 1_024; + +export { + CODE_GRAPH_AUTOMATIC_COMPACTION_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_DEFERRED_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_FAILURE_COOLDOWN_MILLISECONDS, + CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_BYTES, + CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_COOLDOWN_MILLISECONDS, + claimCodeGraphAutomaticCompactionCandidate, + codeGraphAutomaticCompactionCandidateAllowed, + codeGraphAutomaticCompactionCooldownMilliseconds, + recordCodeGraphAutomaticCompactionAttempt, +}; + +export interface CodeGraphAutomaticCompactionCandidate { + readonly checkoutId: string; + readonly opportunityBytes: number; + readonly opportunityRatio: number; +} + +export interface CodeGraphAutomaticCompactionResult { + readonly action: CodeGraphCompactionSummary['action']; + readonly checkoutId: string; + readonly reason?: CodeGraphCompactionSummary['reason']; + readonly reclaimedBytes: number; +} + +export type CodeGraphAutomaticCompactionStatus = + | {readonly state: 'idle'} + | {readonly startedAt: string; readonly state: 'inspecting'} + | { + readonly checkoutId: string; + readonly opportunityBytes: number; + readonly startedAt: string; + readonly state: 'running'; + } + | { + readonly action: 'no-candidate' | 'not-needed' | 'missing'; + readonly completedAt: string; + readonly inspected: number; + readonly inspectionFailures: number; + readonly state: 'completed'; + } + | { + readonly action: 'compacted'; + readonly checkoutId: string; + readonly completedAt: string; + readonly reclaimedBytes: number; + readonly startedAt: string; + readonly state: 'completed'; + } + | { + readonly checkoutId: string; + readonly completedAt: string; + readonly reason: 'active-build' | 'active-maintenance'; + readonly startedAt: string; + readonly state: 'deferred'; + } + | { + readonly checkoutId?: string; + readonly completedAt: string; + readonly reason: 'compaction-failed' | 'inspection-failed'; + readonly startedAt: string; + readonly state: 'failed'; + }; + +interface CodeGraphAutomaticCompactionWorkerRequest { + readonly checkoutId: string; + readonly force: boolean; + readonly operation: 'compact' | 'probe'; + readonly protocol: 1; + readonly threadnoteHome: string; +} + +type CodeGraphAutomaticCompactionWorkerResponse = + | {readonly ok: false; readonly protocol: 1} + | {readonly ok: true; readonly protocol: 1; readonly result: CodeGraphAutomaticCompactionResult}; + +function automaticCompactionHasDiskHeadroom(storage: CodeGraphActiveStorage): boolean { + return ( + storage.availableBytes !== undefined && storage.availableBytes >= codeGraphCompactionRequiredFreeBytes(storage) + ); +} + +export interface CodeGraphAutomaticCompactionDependencies { + readonly candidateAllowed?: ( + threadnoteHome: string, + candidate: CodeGraphAutomaticCompactionCandidate, + ) => Effect.Effect; + readonly compact: ( + threadnoteHome: string, + checkoutId: string, + ) => Effect.Effect; + readonly claimCandidate?: ( + threadnoteHome: string, + candidate: CodeGraphAutomaticCompactionCandidate, + ) => Effect.Effect; + readonly inspect: ( + threadnoteHome: string, + checkoutId: string, + ) => Effect.Effect; + readonly listCheckoutIds: (threadnoteHome: string) => Effect.Effect; + readonly onCandidate?: (candidate: CodeGraphAutomaticCompactionCandidate) => Effect.Effect; + readonly recordAttempt?: ( + threadnoteHome: string, + candidate: CodeGraphAutomaticCompactionCandidate, + result: CodeGraphAutomaticCompactionResult | undefined, + ) => Effect.Effect; +} + +export type CodeGraphAutomaticCompactionPassResult = + | { + readonly inspected: number; + readonly inspectionFailures: number; + readonly nextOffset: number; + readonly state: 'no-candidate'; + } + | { + readonly candidate: CodeGraphAutomaticCompactionCandidate; + readonly inspected: number; + readonly inspectionFailures: number; + readonly nextOffset: number; + readonly result: CodeGraphAutomaticCompactionResult; + readonly state: 'attempted'; + }; + +/** Select one largest reviewed reclaim opportunity; ties are stable across enumeration order. */ +export function selectCodeGraphAutomaticCompactionCandidate( + candidates: readonly CodeGraphAutomaticCompactionCandidate[], +): CodeGraphAutomaticCompactionCandidate | undefined { + return [...candidates].sort( + (left, right) => + right.opportunityBytes - left.opportunityBytes || + right.opportunityRatio - left.opportunityRatio || + compareCodeUnits(left.checkoutId, right.checkoutId), + )[0]; +} + +/** @internal Rotate by one so every database eventually enters the bounded inspection window. */ +export function codeGraphAutomaticCompactionCheckoutWindow( + allCheckoutIds: readonly string[], + offset: number, +): {readonly checkoutIds: readonly string[]; readonly nextOffset: number} { + if (allCheckoutIds.length === 0) return {checkoutIds: [], nextOffset: 0}; + const normalizedOffset = Number.isSafeInteger(offset) && offset >= 0 ? offset % allCheckoutIds.length : 0; + const rotatedCheckoutIds = [...allCheckoutIds.slice(normalizedOffset), ...allCheckoutIds.slice(0, normalizedOffset)]; + return { + checkoutIds: rotatedCheckoutIds.slice(0, CODE_GRAPH_AUTOMATIC_COMPACTION_DATABASE_LIMIT), + nextOffset: (normalizedOffset + 1) % allCheckoutIds.length, + }; +} + +export const runCodeGraphAutomaticCompactionPassWith = Effect.fn('codeGraph.automaticCompactionPassWith')(function* ( + dependencies: CodeGraphAutomaticCompactionDependencies, + threadnoteHome: string, + options: {readonly offset?: number} = {}, +) { + const allCheckoutIds = [...new Set(yield* dependencies.listCheckoutIds(threadnoteHome))] + .filter(checkoutId => /^[0-9a-f]{64}$/u.test(checkoutId)) + .sort(compareCodeUnits); + const offset = + allCheckoutIds.length === 0 || !Number.isSafeInteger(options.offset) || (options.offset ?? 0) < 0 + ? 0 + : (options.offset ?? 0) % allCheckoutIds.length; + const {checkoutIds, nextOffset} = codeGraphAutomaticCompactionCheckoutWindow(allCheckoutIds, offset); + const observations = yield* Effect.forEach( + checkoutIds, + checkoutId => + dependencies.inspect(threadnoteHome, checkoutId).pipe( + Effect.map(storage => ({checkoutId, storage})), + Effect.catch(() => Effect.succeed(undefined)), + ), + {concurrency: 2}, + ); + const inspectionFailures = observations.filter(observation => observation === undefined).length; + const rankedCandidates = [ + ...observations.flatMap(observation => { + if ( + observation === undefined || + observation.storage.state !== 'available' || + observation.storage.pageStorage.state !== 'available' || + observation.storage.pageStorage.threshold.reason !== 'freelist' || + !automaticCompactionHasDiskHeadroom(observation.storage) + ) { + return []; + } + return [ + { + checkoutId: observation.checkoutId, + opportunityBytes: observation.storage.pageStorage.reclaimableBytes, + opportunityRatio: observation.storage.pageStorage.reclaimableRatio, + } satisfies CodeGraphAutomaticCompactionCandidate, + ]; + }), + ].sort( + (left, right) => + right.opportunityBytes - left.opportunityBytes || + right.opportunityRatio - left.opportunityRatio || + compareCodeUnits(left.checkoutId, right.checkoutId), + ); + let candidate: CodeGraphAutomaticCompactionCandidate | undefined; + for (const ranked of rankedCandidates) { + if (yield* dependencies.candidateAllowed?.(threadnoteHome, ranked) ?? Effect.succeed(true)) { + candidate = ranked; + break; + } + } + const inspected = checkoutIds.length - inspectionFailures; + if (candidate === undefined) { + return { + inspected, + inspectionFailures, + nextOffset, + state: 'no-candidate', + } satisfies CodeGraphAutomaticCompactionPassResult; + } + const claimed = yield* dependencies.claimCandidate?.(threadnoteHome, candidate) ?? Effect.succeed(true); + if (!claimed) { + return { + inspected, + inspectionFailures, + nextOffset, + state: 'no-candidate', + } satisfies CodeGraphAutomaticCompactionPassResult; + } + yield* dependencies.onCandidate?.(candidate) ?? Effect.void; + const result = yield* dependencies + .compact(threadnoteHome, candidate.checkoutId) + .pipe(Effect.tapError(() => dependencies.recordAttempt?.(threadnoteHome, candidate, undefined) ?? Effect.void)); + yield* dependencies.recordAttempt?.(threadnoteHome, candidate, result) ?? Effect.void; + return { + candidate, + inspected, + inspectionFailures, + nextOffset, + result, + state: 'attempted', + } satisfies CodeGraphAutomaticCompactionPassResult; +}); + +/** Run synchronous SQLite VACUUM in a killable child so Manager's event loop remains responsive. */ +export const compactCodeGraphStorageIsolated: ( + threadnoteHome: string, + checkoutId: string, + options?: {readonly force?: boolean; readonly operation?: 'compact' | 'probe'}, +) => Effect.Effect< + CodeGraphAutomaticCompactionResult, + CodeGraphAutomaticCompactionError | CommandExecutionError, + CommandExecutor | SystemInfo +> = Effect.fn('codeGraph.compactStorageIsolated')(function* ( + threadnoteHome: string, + checkoutId: string, + options: {readonly force?: boolean; readonly operation?: 'compact' | 'probe'} = {}, +) { + const command = yield* CommandExecutor; + const system = yield* SystemInfo; + const invocation = codeGraphAutomaticCompactionWorkerInvocation(system); + const request = { + checkoutId, + force: options.force === true, + operation: options.operation ?? 'compact', + protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL, + threadnoteHome, + } satisfies CodeGraphAutomaticCompactionWorkerRequest; + const environment = automaticCompactionWorkerEnvironment(system.environment(), threadnoteHome); + const result = yield* command.execute(invocation.executable, invocation.arguments, { + env: environment, + input: new TextEncoder().encode(`${JSON.stringify(request)}\n`), + maxOutputBytes: CODE_GRAPH_AUTOMATIC_COMPACTION_OUTPUT_BYTES_MAXIMUM, + timeoutMs: 0, + }); + const response = decodeAutomaticCompactionWorkerResponse(result.stdout, checkoutId); + if (response === undefined || !response.ok) { + return yield* Effect.fail(new CodeGraphAutomaticCompactionError('Isolated code graph compaction failed.')); + } + return response.result; +}); + +/** @internal Preserve only host variables required to bootstrap the same-privilege worker. */ +export function automaticCompactionWorkerEnvironment( + source: NodeJS.ProcessEnv, + threadnoteHome: string, +): NodeJS.ProcessEnv { + const environment: NodeJS.ProcessEnv = { + THREADNOTE_CODE_GRAPH_COMPACTION_WORKER: '1', + THREADNOTE_HOME: threadnoteHome, + }; + for (const key of [ + 'HOME', + 'USERPROFILE', + 'HOMEDRIVE', + 'HOMEPATH', + 'TMPDIR', + 'TMP', + 'TEMP', + 'PATH', + 'PATHEXT', + 'ComSpec', + 'COMSPEC', + 'SystemRoot', + 'SYSTEMROOT', + 'WINDIR', + ] as const) { + const value = source[key]; + if (value !== undefined) environment[key] = value; + } + return environment; +} + +export const runCodeGraphAutomaticCompactionPass = Effect.fn('codeGraph.automaticCompactionPass')(function* ( + threadnoteHome: string, + options: { + readonly offset?: number; + readonly onCandidate?: (candidate: CodeGraphAutomaticCompactionCandidate) => Effect.Effect; + } = {}, +) { + const dependencies = productionAutomaticCompactionDependencies(options.onCandidate); + return yield* runCodeGraphAutomaticCompactionPassWith(dependencies, threadnoteHome, options); +}); + +export const runCodeGraphAutomaticCompactionLoopWith = Effect.fn('codeGraph.automaticCompactionLoopWith')(function* ( + dependencies: CodeGraphAutomaticCompactionDependencies, + threadnoteHome: string, + onStatus: (status: CodeGraphAutomaticCompactionStatus) => Effect.Effect, + timing: { + readonly initialDelayMilliseconds?: number; + readonly intervalMilliseconds?: number; + } = {}, +) { + yield* Effect.sleep(timing.initialDelayMilliseconds ?? CODE_GRAPH_AUTOMATIC_COMPACTION_INITIAL_DELAY_MILLISECONDS); + let offset = 0; + while (true) { + const startedAtMilliseconds = yield* Clock.currentTimeMillis; + const startedAt = new Date(startedAtMilliseconds).toISOString(); + yield* onStatus({startedAt, state: 'inspecting'}); + let attemptedCandidate: CodeGraphAutomaticCompactionCandidate | undefined; + const outcome: CodeGraphAutomaticCompactionPassResult | undefined = yield* runCodeGraphAutomaticCompactionPassWith( + { + ...dependencies, + onCandidate: candidate => + Effect.gen(function* () { + attemptedCandidate = candidate; + yield* dependencies.onCandidate?.(candidate) ?? Effect.void; + yield* onStatus({ + checkoutId: candidate.checkoutId, + opportunityBytes: candidate.opportunityBytes, + startedAt, + state: 'running', + }); + }), + }, + threadnoteHome, + {offset}, + ).pipe(Effect.match({onFailure: () => undefined, onSuccess: result => result})); + const completedAt = new Date(yield* Clock.currentTimeMillis).toISOString(); + if (outcome === undefined) { + yield* onStatus({ + ...(attemptedCandidate === undefined ? {} : {checkoutId: attemptedCandidate.checkoutId}), + completedAt, + reason: attemptedCandidate === undefined ? 'inspection-failed' : 'compaction-failed', + startedAt, + state: 'failed', + }); + } else { + offset = outcome.nextOffset; + if (outcome.state === 'no-candidate') { + yield* onStatus( + outcome.inspected === 0 && outcome.inspectionFailures > 0 + ? {completedAt, reason: 'inspection-failed', startedAt, state: 'failed'} + : { + action: 'no-candidate', + completedAt, + inspected: outcome.inspected, + inspectionFailures: outcome.inspectionFailures, + state: 'completed', + }, + ); + } else if (outcome.result.action === 'deferred') { + yield* onStatus({ + checkoutId: outcome.candidate.checkoutId, + completedAt, + reason: outcome.result.reason ?? 'active-maintenance', + startedAt, + state: 'deferred', + }); + } else if (outcome.result.action === 'compacted') { + yield* onStatus({ + action: 'compacted', + checkoutId: outcome.candidate.checkoutId, + completedAt, + reclaimedBytes: outcome.result.reclaimedBytes, + startedAt, + state: 'completed', + }); + } else { + yield* onStatus({ + action: outcome.result.action === 'would-compact' ? 'not-needed' : outcome.result.action, + completedAt, + inspected: outcome.inspected, + inspectionFailures: outcome.inspectionFailures, + state: 'completed', + }); + } + } + yield* Effect.sleep(timing.intervalMilliseconds ?? CODE_GRAPH_AUTOMATIC_COMPACTION_INTERVAL_MILLISECONDS); + } +}); + +/** + * Manager is a long-lived, user-visible owner for safe opportunistic compaction. + * Each pass attempts at most one database and the storage boundary independently + * fences active builders, maintenance, snapshot receipts, and disk headroom. + */ +export const runCodeGraphAutomaticCompactionLoop = Effect.fn('codeGraph.automaticCompactionLoop')(function* ( + threadnoteHome: string, + onStatus: (status: CodeGraphAutomaticCompactionStatus) => Effect.Effect = () => Effect.void, +) { + return yield* runCodeGraphAutomaticCompactionLoopWith( + productionAutomaticCompactionDependencies(), + threadnoteHome, + onStatus, + ); +}); + +function productionAutomaticCompactionDependencies( + onCandidate?: (candidate: CodeGraphAutomaticCompactionCandidate) => Effect.Effect, +): CodeGraphAutomaticCompactionDependencies< + CommandExecutor | Crypto.Crypto | FileSystem.FileSystem | Path.Path | SystemInfo +> { + return { + candidateAllowed: codeGraphAutomaticCompactionCandidateAllowed, + claimCandidate: claimCodeGraphAutomaticCompactionCandidate, + compact: compactCodeGraphStorageIsolated, + inspect: (home, checkoutId) => inspectCodeGraphStorage(home, checkoutId), + listCheckoutIds: listCodeGraphAutomaticCompactionCheckoutIds, + onCandidate, + recordAttempt: recordCodeGraphAutomaticCompactionAttempt, + }; +} + +/** @internal Bounded inventory; overflow is explicit instead of silently starving tail repositories. */ +export const listCodeGraphAutomaticCompactionCheckoutIds = Effect.fn('codeGraph.listAutomaticCompactionCheckoutIds')( + function* (threadnoteHome: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const repositories = codeGraphRepositoriesRoot(path, threadnoteHome); + if (Option.isSome(yield* fs.readLink(repositories).pipe(Effect.option))) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionError('Code graph repository storage is not a directory.'), + ); + } + if (!(yield* fs.exists(repositories))) return []; + const page = yield* runtimeTextDirectoryNamePage(repositories, CODE_GRAPH_AUTOMATIC_COMPACTION_DATABASE_LIMIT); + if (page.overflow) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionError( + 'Automatic code graph compaction inventory exceeded its bounded repository limit.', + ), + ); + } + const checkoutIds: string[] = []; + for (const checkoutId of page.names.filter(name => /^[0-9a-f]{64}$/u.test(name)).sort(compareCodeUnits)) { + const repositoryRoot = path.join(repositories, checkoutId); + if (Option.isSome(yield* fs.readLink(repositoryRoot).pipe(Effect.option))) continue; + const repositoryInfo = yield* fs.stat(repositoryRoot).pipe(Effect.option); + if (Option.isNone(repositoryInfo) || repositoryInfo.value.type !== 'Directory') continue; + const database = path.join(repositoryRoot, `graph-v${CODE_GRAPH_SCHEMA_VERSION}.sqlite`); + if (Option.isSome(yield* fs.readLink(database).pipe(Effect.option))) continue; + const databaseInfo = yield* fs.stat(database).pipe(Effect.option); + if (Option.isSome(databaseInfo) && databaseInfo.value.type === 'File') checkoutIds.push(checkoutId); + } + return checkoutIds; + }, +); + +/** Internal standalone worker. Synchronous SQLite work stays outside Manager's JS event loop. */ +export const codeGraphAutomaticCompactionWorkerProgram = Effect.gen(function* () { + const stdio = yield* Stdio.Stdio; + const content = yield* readBoundedAutomaticCompactionWorkerInput(stdio); + const request = decodeAutomaticCompactionWorkerRequest(content); + let response: CodeGraphAutomaticCompactionWorkerResponse; + if (request === undefined) { + response = {ok: false, protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL}; + } else if (request.operation === 'probe') { + response = yield* compactCodeGraphStorage(request.threadnoteHome, request.checkoutId, { + dryRun: true, + force: request.force, + }).pipe( + Effect.map(automaticCompactionResult), + Effect.match({onFailure: automaticCompactionWorkerFailure, onSuccess: automaticCompactionWorkerSuccess}), + ); + } else { + response = yield* compactCodeGraphStorage(request.threadnoteHome, request.checkoutId, { + dryRun: false, + force: request.force, + }).pipe( + Effect.map(automaticCompactionResult), + Effect.match({onFailure: automaticCompactionWorkerFailure, onSuccess: automaticCompactionWorkerSuccess}), + ); + } + yield* Stream.run( + Stream.make(new TextEncoder().encode(`${JSON.stringify(response)}\n`)), + stdio.stdout({endOnDone: false}), + ); +}).pipe(Effect.catch(() => Effect.void)); + +function automaticCompactionWorkerFailure(): CodeGraphAutomaticCompactionWorkerResponse { + return {ok: false, protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL}; +} + +function automaticCompactionWorkerSuccess( + result: CodeGraphAutomaticCompactionResult, +): CodeGraphAutomaticCompactionWorkerResponse { + return {ok: true, protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL, result}; +} + +function automaticCompactionResult(summary: CodeGraphCompactionSummary): CodeGraphAutomaticCompactionResult { + return { + action: summary.action, + checkoutId: summary.checkoutId, + ...(summary.reason === undefined ? {} : {reason: summary.reason}), + reclaimedBytes: summary.reclaimedBytes, + }; +} + +function readBoundedAutomaticCompactionWorkerInput(stdio: Stdio.Stdio): Effect.Effect { + const encoder = new TextEncoder(); + return stdio.stdin.pipe( + Stream.decodeText, + Stream.runFoldEffect( + () => ({chunks: [] as string[], size: 0}), + (state, chunk) => { + const size = state.size + encoder.encode(chunk).byteLength; + if (size > CODE_GRAPH_AUTOMATIC_COMPACTION_INPUT_BYTES_MAXIMUM) { + return Effect.fail(new CodeGraphAutomaticCompactionError('Code graph compaction request was too large.')); + } + state.chunks.push(chunk); + return Effect.succeed({chunks: state.chunks, size}); + }, + ), + Effect.map(state => state.chunks.join('')), + ); +} + +/** @internal Strict worker protocol decoder used by focused transport tests. */ +export function decodeAutomaticCompactionWorkerRequest( + content: string, +): CodeGraphAutomaticCompactionWorkerRequest | undefined { + try { + const parsed: unknown = JSON.parse(content.trim()); + if (typeof parsed !== 'object' || parsed === null) return undefined; + const record = parsed as Readonly>; + if ( + record.protocol !== CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL || + typeof record.threadnoteHome !== 'string' || + record.threadnoteHome.length === 0 || + record.threadnoteHome.length > 8_192 || + record.threadnoteHome.includes('\0') || + !automaticCompactionAbsolutePath(record.threadnoteHome) || + typeof record.checkoutId !== 'string' || + !/^[0-9a-f]{64}$/u.test(record.checkoutId) || + typeof record.force !== 'boolean' || + !['compact', 'probe'].includes(String(record.operation)) + ) { + return undefined; + } + return { + checkoutId: record.checkoutId, + force: record.force, + operation: record.operation as CodeGraphAutomaticCompactionWorkerRequest['operation'], + protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL, + threadnoteHome: record.threadnoteHome, + }; + } catch { + return undefined; + } +} + +/** @internal Strict worker protocol decoder used by focused transport tests. */ +export function decodeAutomaticCompactionWorkerResponse( + content: string, + expectedCheckoutId?: string, +): CodeGraphAutomaticCompactionWorkerResponse | undefined { + try { + const parsed: unknown = JSON.parse(content.trim()); + if (typeof parsed !== 'object' || parsed === null) return undefined; + const record = parsed as Readonly>; + if (record.protocol !== CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL || typeof record.ok !== 'boolean') { + return undefined; + } + if (!record.ok) return {ok: false, protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL}; + if (typeof record.result !== 'object' || record.result === null) return undefined; + const result = record.result as Readonly>; + if ( + !['compacted', 'deferred', 'missing', 'not-needed', 'would-compact'].includes(String(result.action)) || + typeof result.checkoutId !== 'string' || + !/^[0-9a-f]{64}$/u.test(result.checkoutId) || + (expectedCheckoutId !== undefined && result.checkoutId !== expectedCheckoutId) || + typeof result.reclaimedBytes !== 'number' || + !Number.isSafeInteger(result.reclaimedBytes) || + result.reclaimedBytes < 0 || + (result.reason !== undefined && !['active-build', 'active-maintenance'].includes(String(result.reason))) || + (result.action === 'deferred' ? result.reason === undefined : result.reason !== undefined) + ) { + return undefined; + } + return { + ok: true, + protocol: CODE_GRAPH_AUTOMATIC_COMPACTION_PROTOCOL, + result: { + action: result.action as CodeGraphAutomaticCompactionResult['action'], + checkoutId: result.checkoutId, + ...(result.reason === undefined + ? {} + : {reason: result.reason as NonNullable}), + reclaimedBytes: result.reclaimedBytes, + }, + }; + } catch { + return undefined; + } +} + +function automaticCompactionAbsolutePath(value: string): boolean { + return value.startsWith('/') || value.startsWith('\\\\') || /^[A-Za-z]:[/\\]/u.test(value); +} + +/** @internal Re-invoke either the compiled binary or the current development standalone. */ +export function codeGraphAutomaticCompactionWorkerInvocation( + system: Pick, +): { + readonly arguments: readonly string[]; + readonly executable: string; +} { + const executableName = system.executablePath.replaceAll('\\', '/').split('/').at(-1)?.toLowerCase(); + if (executableName !== 'bun' && executableName !== 'bun.exe') { + return {arguments: [CODE_GRAPH_COMPACTION_WORKER_ARGUMENT], executable: system.executablePath}; + } + const currentScript = system.processArguments[1]; + const standaloneScript = + currentScript && /(?:^|[/\\])(?:standalone\.(?:js|ts)|threadnote\.cjs)$/iu.test(currentScript) + ? currentScript + : Bun.fileURLToPath(new URL('../standalone.ts', import.meta.url)); + return { + arguments: [standaloneScript, CODE_GRAPH_COMPACTION_WORKER_ARGUMENT], + executable: system.executablePath, + }; +} diff --git a/src/code_graph/automatic_compaction_receipt.ts b/src/code_graph/automatic_compaction_receipt.ts new file mode 100644 index 00000000..b6983d83 --- /dev/null +++ b/src/code_graph/automatic_compaction_receipt.ts @@ -0,0 +1,334 @@ +import {Clock, Crypto, Effect, FileSystem, Option, Path} from 'effect'; +import {syncDirectoryBestEffort, syncWritableFile} from '../effect/file_durability.js'; +import {withExclusiveFileLock} from '../effect/file_lock.js'; +import {codeGraphRepositoriesRoot, codeGraphRepositoryRoot} from './layout.js'; + +class CodeGraphAutomaticCompactionReceiptError extends Error { + readonly _tag = 'CodeGraphAutomaticCompactionReceiptError' as const; +} + +export const CODE_GRAPH_AUTOMATIC_COMPACTION_COOLDOWN_MILLISECONDS = 24 * 60 * 60 * 1_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_COOLDOWN_MILLISECONDS = 7 * 24 * 60 * 60 * 1_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_BYTES = 64 * 1_024 * 1_024; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_FAILURE_COOLDOWN_MILLISECONDS = 60 * 60 * 1_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_DEFERRED_COOLDOWN_MILLISECONDS = 5 * 60 * 1_000; +export const CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_FILE = 'automatic-compaction-v1.json'; +const CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_BYTES_MAXIMUM = 4 * 1_024; +const CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_LOCK_OPTIONS = { + retryIntervalMilliseconds: 25, + staleAfterMilliseconds: 60 * 60 * 1_000, + waitTimeoutMilliseconds: 0, +} as const; + +export interface CodeGraphAutomaticCompactionReceiptCandidate { + readonly checkoutId: string; + readonly opportunityBytes: number; +} + +export interface CodeGraphAutomaticCompactionReceiptResult { + readonly action: 'compacted' | 'deferred' | 'missing' | 'not-needed' | 'would-compact'; + readonly reclaimedBytes: number; +} + +type CodeGraphAutomaticCompactionReceiptAction = + 'attempting' | 'compacted' | 'deferred' | 'failed' | 'missing' | 'not-needed'; + +interface CodeGraphAutomaticCompactionReceipt { + readonly action: CodeGraphAutomaticCompactionReceiptAction; + readonly checkoutId: string; + readonly opportunityBytes: number; + readonly reclaimedBytes: number; + readonly recordedAtMilliseconds: number; + readonly retryAfterMilliseconds: number; + readonly version: 1; +} + +/** Stable policy shared by automatic and explicit compaction paths. */ +export function codeGraphAutomaticCompactionCooldownMilliseconds( + result: CodeGraphAutomaticCompactionReceiptResult | undefined, +): number { + if (result === undefined) return CODE_GRAPH_AUTOMATIC_COMPACTION_FAILURE_COOLDOWN_MILLISECONDS; + if (result.action === 'deferred') return CODE_GRAPH_AUTOMATIC_COMPACTION_DEFERRED_COOLDOWN_MILLISECONDS; + if (result.action !== 'compacted') return CODE_GRAPH_AUTOMATIC_COMPACTION_FAILURE_COOLDOWN_MILLISECONDS; + return result.reclaimedBytes < CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_BYTES + ? CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_COOLDOWN_MILLISECONDS + : CODE_GRAPH_AUTOMATIC_COMPACTION_COOLDOWN_MILLISECONDS; +} + +/** Read the shared private receipt without claiming the candidate. */ +export const codeGraphAutomaticCompactionCandidateAllowed = Effect.fn('codeGraph.automaticCompactionCandidateAllowed')( + function* (threadnoteHome: string, candidate: CodeGraphAutomaticCompactionReceiptCandidate) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const now = yield* Clock.currentTimeMillis; + return yield* automaticCompactionReceiptAllows(fs, path, threadnoteHome, candidate.checkoutId, now); + }, +); + +/** Atomically reserve one candidate across all local Manager processes. */ +export const claimCodeGraphAutomaticCompactionCandidate = Effect.fn('codeGraph.claimAutomaticCompactionCandidate')( + function* (threadnoteHome: string, candidate: CodeGraphAutomaticCompactionReceiptCandidate) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const lockPath = automaticCompactionReceiptLockPath(path, threadnoteHome, candidate.checkoutId); + return yield* withExclusiveFileLock( + fs, + lockPath, + CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_LOCK_OPTIONS, + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (!(yield* automaticCompactionReceiptAllows(fs, path, threadnoteHome, candidate.checkoutId, now))) { + return false; + } + yield* writeAutomaticCompactionReceipt(fs, path, threadnoteHome, { + action: 'attempting', + checkoutId: candidate.checkoutId, + opportunityBytes: candidate.opportunityBytes, + reclaimedBytes: 0, + recordedAtMilliseconds: now, + retryAfterMilliseconds: now + CODE_GRAPH_AUTOMATIC_COMPACTION_COOLDOWN_MILLISECONDS, + version: 1, + }); + return true; + }), + ).pipe(Effect.catch(() => Effect.succeed(false))); + }, +); + +/** Finalize a claim; a write failure leaves the conservative attempting receipt intact. */ +export const recordCodeGraphAutomaticCompactionAttempt = Effect.fn('codeGraph.recordAutomaticCompactionAttempt')( + function* ( + threadnoteHome: string, + candidate: CodeGraphAutomaticCompactionReceiptCandidate, + result: CodeGraphAutomaticCompactionReceiptResult | undefined, + ) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const lockPath = automaticCompactionReceiptLockPath(path, threadnoteHome, candidate.checkoutId); + yield* withExclusiveFileLock( + fs, + lockPath, + CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_LOCK_OPTIONS, + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + yield* writeAutomaticCompactionReceipt(fs, path, threadnoteHome, { + action: automaticCompactionReceiptAction(result), + checkoutId: candidate.checkoutId, + opportunityBytes: candidate.opportunityBytes, + reclaimedBytes: result?.reclaimedBytes ?? 0, + recordedAtMilliseconds: now, + retryAfterMilliseconds: now + codeGraphAutomaticCompactionCooldownMilliseconds(result), + version: 1, + }); + }), + ).pipe(Effect.catch(() => Effect.void)); + }, +); + +function automaticCompactionReceiptAction( + result: CodeGraphAutomaticCompactionReceiptResult | undefined, +): CodeGraphAutomaticCompactionReceiptAction { + if (result === undefined) return 'failed'; + if (result.action === 'compacted' || result.action === 'deferred' || result.action === 'missing') { + return result.action; + } + return 'not-needed'; +} + +function automaticCompactionReceiptLockPath(path: Path.Path, threadnoteHome: string, checkoutId: string): string { + return path.join(threadnoteHome, 'locks', 'indexes', 'code-graph', 'automatic-compaction', `${checkoutId}.lock`); +} + +function automaticCompactionReceiptAllows( + fs: FileSystem.FileSystem, + path: Path.Path, + threadnoteHome: string, + checkoutId: string, + now: number, +): Effect.Effect { + return Effect.gen(function* () { + const authority = yield* inspectAutomaticCompactionReceiptRoot(fs, path, threadnoteHome, checkoutId); + const target = path.join(authority.root, CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_FILE); + if (!(yield* fs.exists(target))) return true; + if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) return false; + const info = yield* fs.stat(target); + if (info.type !== 'File') return false; + const modifiedAt = Option.getOrUndefined(info.mtime)?.getTime(); + const malformedAllowed = + modifiedAt !== undefined && now - modifiedAt >= CODE_GRAPH_AUTOMATIC_COMPACTION_LOW_YIELD_COOLDOWN_MILLISECONDS; + if (Number(info.size) > CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_BYTES_MAXIMUM) return malformedAllowed; + const content = yield* fs.readFileString(target); + if (new TextEncoder().encode(content).byteLength > CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_BYTES_MAXIMUM) { + return malformedAllowed; + } + const receipt = decodeAutomaticCompactionReceipt(content, checkoutId); + return receipt === undefined ? malformedAllowed : now >= receipt.retryAfterMilliseconds; + }).pipe(Effect.catch(() => Effect.succeed(false))); +} + +const writeAutomaticCompactionReceipt = Effect.fn('codeGraph.writeAutomaticCompactionReceipt')(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + threadnoteHome: string, + receipt: CodeGraphAutomaticCompactionReceipt, +) { + const crypto = yield* Crypto.Crypto; + const authority = yield* inspectAutomaticCompactionReceiptRoot(fs, path, threadnoteHome, receipt.checkoutId); + const root = authority.root; + const target = path.join(root, CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_FILE); + if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) { + return yield* Effect.fail(new CodeGraphAutomaticCompactionReceiptError('Compaction receipt is not a file.')); + } + const content = `${JSON.stringify(receipt)}\n`; + if (new TextEncoder().encode(content).byteLength > CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_BYTES_MAXIMUM) { + return yield* Effect.fail(new CodeGraphAutomaticCompactionReceiptError('Compaction receipt is too large.')); + } + const temporary = path.join( + root, + `.${CODE_GRAPH_AUTOMATIC_COMPACTION_RECEIPT_FILE}.${yield* crypto.randomUUIDv4}.tmp`, + ); + yield* Effect.gen(function* () { + yield* fs.writeFileString(temporary, content, {flag: 'wx', mode: 0o600}); + yield* syncWritableFile(fs, temporary); + const revalidated = yield* inspectAutomaticCompactionReceiptRoot(fs, path, threadnoteHome, receipt.checkoutId); + if (!sameAutomaticCompactionReceiptRoot(authority, revalidated)) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionReceiptError('Compaction receipt directory changed during publication.'), + ); + } + if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) { + return yield* Effect.fail(new CodeGraphAutomaticCompactionReceiptError('Compaction receipt is not a file.')); + } + yield* fs.rename(temporary, target); + yield* syncDirectoryBestEffort(fs, root); + }).pipe(Effect.ensuring(fs.remove(temporary, {force: true}).pipe(Effect.catch(() => Effect.void)))); +}); + +interface AutomaticCompactionReceiptRoot { + readonly dev: number; + readonly ino?: number; + readonly parentDev: number; + readonly parentIno?: number; + readonly root: string; +} + +function inspectAutomaticCompactionReceiptRoot( + fs: FileSystem.FileSystem, + path: Path.Path, + threadnoteHome: string, + checkoutId: string, +): Effect.Effect { + return Effect.gen(function* () { + const declaredRoot = codeGraphRepositoryRoot(path, threadnoteHome, checkoutId); + const declaredParent = codeGraphRepositoriesRoot(path, threadnoteHome); + if (Option.isSome(yield* fs.readLink(declaredParent).pipe(Effect.option))) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionReceiptError('Compaction repositories directory is a symbolic link.'), + ); + } + if (Option.isSome(yield* fs.readLink(declaredRoot).pipe(Effect.option))) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionReceiptError('Compaction receipt directory is a symbolic link.'), + ); + } + const canonicalHome = yield* fs.realPath(threadnoteHome); + const canonicalParent = yield* fs.realPath(declaredParent); + const root = yield* fs.realPath(declaredRoot); + const parentInfo = yield* fs.stat(canonicalParent); + const info = yield* fs.stat(root); + if ( + parentInfo.type !== 'Directory' || + info.type !== 'Directory' || + canonicalParent !== path.join(canonicalHome, 'indexes', 'code-graph', 'repositories') || + path.dirname(root) !== canonicalParent || + path.basename(root) !== checkoutId + ) { + return yield* Effect.fail( + new CodeGraphAutomaticCompactionReceiptError('Compaction receipt directory escaped graph storage.'), + ); + } + return { + dev: info.dev, + ...(Option.isSome(info.ino) ? {ino: info.ino.value} : {}), + parentDev: parentInfo.dev, + ...(Option.isSome(parentInfo.ino) ? {parentIno: parentInfo.ino.value} : {}), + root, + }; + }).pipe( + Effect.mapError( + cause => + new CodeGraphAutomaticCompactionReceiptError('Could not safely inspect compaction receipt storage.', {cause}), + ), + ); +} + +function sameAutomaticCompactionReceiptRoot( + left: AutomaticCompactionReceiptRoot, + right: AutomaticCompactionReceiptRoot, +): boolean { + return ( + left.root === right.root && + left.dev === right.dev && + (left.ino === undefined || right.ino === undefined || left.ino === right.ino) && + left.parentDev === right.parentDev && + (left.parentIno === undefined || right.parentIno === undefined || left.parentIno === right.parentIno) + ); +} + +function decodeAutomaticCompactionReceipt( + content: string, + expectedCheckoutId: string, +): CodeGraphAutomaticCompactionReceipt | undefined { + try { + const parsed: unknown = JSON.parse(content); + if (typeof parsed !== 'object' || parsed === null) return undefined; + const receipt = parsed as Readonly>; + const keys = Object.keys(receipt).sort(); + if ( + JSON.stringify(keys) !== + JSON.stringify( + [ + 'action', + 'checkoutId', + 'opportunityBytes', + 'reclaimedBytes', + 'recordedAtMilliseconds', + 'retryAfterMilliseconds', + 'version', + ].sort(), + ) || + receipt.version !== 1 || + receipt.checkoutId !== expectedCheckoutId || + !automaticCompactionReceiptActionValue(receipt.action) || + !automaticCompactionReceiptInteger(receipt.opportunityBytes) || + !automaticCompactionReceiptInteger(receipt.reclaimedBytes) || + !automaticCompactionReceiptInteger(receipt.recordedAtMilliseconds) || + !automaticCompactionReceiptInteger(receipt.retryAfterMilliseconds) || + receipt.retryAfterMilliseconds < receipt.recordedAtMilliseconds + ) { + return undefined; + } + return { + action: receipt.action, + checkoutId: receipt.checkoutId, + opportunityBytes: receipt.opportunityBytes, + reclaimedBytes: receipt.reclaimedBytes, + recordedAtMilliseconds: receipt.recordedAtMilliseconds, + retryAfterMilliseconds: receipt.retryAfterMilliseconds, + version: 1, + }; + } catch { + return undefined; + } +} + +function automaticCompactionReceiptActionValue(value: unknown): value is CodeGraphAutomaticCompactionReceiptAction { + return ( + typeof value === 'string' && + ['attempting', 'compacted', 'deferred', 'failed', 'missing', 'not-needed'].includes(value) + ); +} + +function automaticCompactionReceiptInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} diff --git a/src/code_graph/build_status.ts b/src/code_graph/build_status.ts index 3f42ef26..c551b963 100644 --- a/src/code_graph/build_status.ts +++ b/src/code_graph/build_status.ts @@ -3,6 +3,14 @@ import {sha256HexSync} from '../crypto/sha256.js'; import {readExclusiveFileLockOwner, type FileLockOwner} from '../effect/file_lock.js'; import {runtimeTextDirectoryNamePage, SystemInfo, type SystemInfoShape} from '../effect/system.js'; import type {CodeGraphBuildOwnerIdentity} from './build_owner.js'; +import {parseCodeGraphBuildStatus} from './build_status_codec.js'; +import { + CODE_GRAPH_BUILD_HASH_ID as HASH_ID, + CODE_GRAPH_BUILD_ID as BUILD_ID, + CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION, + isBuildStatusRecord as isRecord, + isBuildStatusText as isText, +} from './build_status_validation.js'; import {classifyCodeGraphLifecycle, type CodeGraphLifecycleProtection} from './lifecycle_classification.js'; import {codeGraphRepositoriesRoot, codeGraphWorktreeLockPath, type CodeGraphLayout} from './layout.js'; import { @@ -18,7 +26,6 @@ import { CODE_GRAPH_TOP_SLOW_FILE_LIMIT, codeGraphPathExtension, codeGraphSourceSizeBucket, - isCodeGraphSourceSizeBucket, retainCodeGraphSlowFileTelemetry, type CodeGraphScanningMetrics, type CodeGraphSlowFileTelemetry, @@ -29,19 +36,22 @@ import type { CodeGraphIndexSummary, CodeGraphMaterializationActivity, CodeGraphMaterializationMetrics, - CodeGraphMaterializationRows, - CodeGraphOverlayFallbackReason, CodeGraphProgress, CodeGraphResolutionActivity, CodeGraphSnapshot, RepositoryIdentity, } from './types.js'; -export const CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION = 1 as const; +export {parseCodeGraphBuildStatus} from './build_status_codec.js'; +export {CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION} from './build_status_validation.js'; export const CODE_GRAPH_BUILD_HEARTBEAT_INTERVAL_MILLISECONDS = 2_000; export const CODE_GRAPH_BUILD_PROGRESS_WRITE_INTERVAL_MILLISECONDS = 250; export const CODE_GRAPH_BUILD_STALE_AFTER_MILLISECONDS = 15_000; +class CodeGraphBuildStatusError extends Error { + readonly _tag = 'CodeGraphBuildStatusError' as const; +} + export type CodeGraphBuildState = 'completed' | 'failed' | 'queued' | 'running'; export type CodeGraphBuildLiveness = 'abandoned' | 'active' | 'completed' | 'failed' | 'stalled'; @@ -167,6 +177,7 @@ export interface ObservedCodeGraphBuildStatus extends CodeGraphBuildStatus { }; /** Local-only Manager context. Never written into the privacy-safe build status document. */ readonly managerContext?: { + readonly branch?: string; readonly worktreePath: string; }; readonly observation: { @@ -223,39 +234,7 @@ const MANAGER_CONTEXT_FILE_BYTES_LIMIT = 8 * 1_024; const MANAGER_CONTEXT_SCHEMA_VERSION = 1 as const; const BUILD_HISTORY_INVALID_RETRY_MILLISECONDS = 30_000; const BUILD_HISTORY_IO_RETRY_MILLISECONDS = 1_000; -const HASH_ID = /^[0-9a-f]{64}$/; -const BUILD_ID = /^[0-9a-f-]{16,64}$/; const BUILD_STATUS_FILE = /^([0-9a-f-]{16,64})\.json$/; -const COMMIT_ID = /^[0-9a-f]{7,64}$/; -const VALID_PHASES = new Set([ - 'activating', - 'embedding', - 'materializing', - 'reclaiming', - 'registering', - 'resolving', - 'scanning', - 'waiting', -]); -const VALID_STATES = new Set(['completed', 'failed', 'queued', 'running']); -const VALID_MATERIALIZATION_FALLBACK_REASONS = new Set([ - 'cache-incomplete', - 'disabled', - 'dynamic-aliases', - 'extractor-context-changed', - 'fact-budget-expanded', - 'file-set-changed', - 'forced-full-rebuild', - 'incremental-rewrite-unbounded', - 'no-materialized-changes', - 'project-closure-incomplete', - 'project-closure-unbounded', - 'reexport-closure-unbounded', - 'resolution-surface-changed', - 'staging-identity-mismatch', - 'staging-unavailable', - 'workspace-changed', -]); export type CodeGraphBuildHistoryPruneResult = | {readonly state: 'complete'} @@ -359,7 +338,9 @@ export const makeCodeGraphBuildReporter = Effect.fn('codeGraph.buildStatus.makeR .pipe(Effect.catch(() => Effect.void)); yield* persist(current => current, true); - yield* writeCodeGraphManagerContext(fs, path, file, buildId, identity.repoRoot).pipe(Effect.catch(() => Effect.void)); + yield* writeCodeGraphManagerContext(fs, path, file, buildId, identity.repoRoot, identity.branch).pipe( + Effect.catch(() => Effect.void), + ); reporterHistoryAuthority.current = Option.getOrUndefined( yield* inspectBuildHistoryDirectory(fs, path, layout, identity.worktreeId).pipe(Effect.option), ); @@ -484,7 +465,7 @@ export const makeCodeGraphBuildReporter = Effect.fn('codeGraph.buildStatus.makeR }; }, true); } - }).pipe(Effect.catch(() => Effect.void)), + }), ownerIdentity: { buildId, processId: system.processId, @@ -982,7 +963,8 @@ function codeGraphBuildStatusPath( worktreeId: string, buildId: string, ): string { - if (!HASH_ID.test(worktreeId) || !BUILD_ID.test(buildId)) throw new Error('Code graph build identity is invalid.'); + if (!HASH_ID.test(worktreeId) || !BUILD_ID.test(buildId)) + throw new CodeGraphBuildStatusError('Code graph build identity is invalid.'); return path.join(layout.repositoryRoot, STATUS_DIRECTORY, worktreeId, `${buildId}.json`); } @@ -999,15 +981,17 @@ function writeCodeGraphBuildStatus( if (initializeDirectory) { yield* ensurePrivateRegularDirectory(fs, path, directory); } else if (!(yield* regularDirectory(fs, directory))) { - return yield* Effect.fail(new Error('Code graph build status directory was removed.')); + return yield* Effect.fail(new CodeGraphBuildStatusError('Code graph build status directory was removed.')); } if ((yield* fs.readLink(file).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph build status path is a symbolic link.')); + return yield* Effect.fail(new CodeGraphBuildStatusError('Code graph build status path is a symbolic link.')); } const temporary = path.join(directory, `.${status.buildId}.${sequence}.tmp`); const content = `${JSON.stringify(status)}\n`; if (new TextEncoder().encode(content).byteLength > STATUS_FILE_BYTES_LIMIT) { - return yield* Effect.fail(new Error('Code graph build status exceeded its bounded sidecar size.')); + return yield* Effect.fail( + new CodeGraphBuildStatusError('Code graph build status exceeded its bounded sidecar size.'), + ); } yield* fs.writeFileString(temporary, content, {flag: 'wx', mode: 0o600}); yield* fs @@ -1026,13 +1010,19 @@ function writeCodeGraphManagerContext( statusFile: string, buildId: string, worktreePath: string, + branch?: string, ) { return Effect.gen(function* () { if (!isText(worktreePath, 4_096)) return; const file = codeGraphManagerContextPath(path, statusFile, buildId); if ((yield* fs.readLink(file).pipe(Effect.option))._tag === 'Some') return; const temporary = path.join(path.dirname(file), `.${buildId}.manager-context.tmp`); - const content = `${JSON.stringify({buildId, schemaVersion: MANAGER_CONTEXT_SCHEMA_VERSION, worktreePath})}\n`; + const content = `${JSON.stringify({ + ...(branch !== undefined && isText(branch, 1_024) ? {branch} : {}), + buildId, + schemaVersion: MANAGER_CONTEXT_SCHEMA_VERSION, + worktreePath, + })}\n`; if (new TextEncoder().encode(content).byteLength > MANAGER_CONTEXT_FILE_BYTES_LIMIT) return; yield* fs.writeFileString(temporary, content, {flag: 'wx', mode: 0o600}); yield* fs @@ -1098,11 +1088,12 @@ function readCodeGraphManagerContext(fs: FileSystem.FileSystem, file: string, bu !isRecord(value) || value.schemaVersion !== MANAGER_CONTEXT_SCHEMA_VERSION || value.buildId !== buildId || - !isText(value.worktreePath, 4_096) + !isText(value.worktreePath, 4_096) || + (value.branch !== undefined && !isText(value.branch, 1_024)) ) { return undefined; } - return {worktreePath: value.worktreePath}; + return {...(value.branch === undefined ? {} : {branch: value.branch}), worktreePath: value.worktreePath}; }).pipe(Effect.catch(() => Effect.succeed(undefined))); } @@ -1111,11 +1102,11 @@ function ensurePrivateRegularDirectory(fs: FileSystem.FileSystem, path: Path.Pat const parent = path.dirname(directory); yield* fs.makeDirectory(parent, {recursive: true, mode: 0o700}); if ((yield* fs.readLink(parent).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph build status parent is a symbolic link.')); + return yield* Effect.fail(new CodeGraphBuildStatusError('Code graph build status parent is a symbolic link.')); } yield* fs.makeDirectory(directory, {recursive: true, mode: 0o700}); if ((yield* fs.readLink(directory).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph build status directory is a symbolic link.')); + return yield* Effect.fail(new CodeGraphBuildStatusError('Code graph build status directory is a symbolic link.')); } }); } @@ -1168,791 +1159,6 @@ function readStatusFile(fs: FileSystem.FileSystem, file: string) { }).pipe(Effect.catch(() => Effect.succeed(undefined))); } -export function parseCodeGraphBuildStatus(value: unknown): CodeGraphBuildStatus | undefined { - if (!isRecord(value) || value.schemaVersion !== CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION) return undefined; - if (!isText(value.buildId, 64) || !BUILD_ID.test(value.buildId)) return undefined; - if (!isRecord(value.identity) || !isRecord(value.owner) || !isRecord(value.timestamps)) return undefined; - if ( - !isHash(value.identity.repositoryId) || - !isHash(value.identity.checkoutId) || - !isHash(value.identity.worktreeId) || - !isText(value.identity.commit, 64) || - !COMMIT_ID.test(value.identity.commit) || - !Number.isSafeInteger(value.owner.processId) || - Number(value.owner.processId) <= 0 || - value.owner.runtime !== 'bun' || - !isText(value.owner.runtimeVersion, 64) || - !VALID_PHASES.has(value.phase as CodeGraphProgress['phase']) || - !VALID_STATES.has(value.state as CodeGraphBuildState) - ) { - return undefined; - } - const timestamps = value.timestamps; - if ( - !isTimestamp(timestamps.startedAt) || - !isTimestamp(timestamps.phaseStartedAt) || - !isTimestamp(timestamps.lastProgressAt) || - !isTimestamp(timestamps.heartbeatAt) || - !isTimestamp(timestamps.updatedAt) || - (timestamps.completedAt !== undefined && !isTimestamp(timestamps.completedAt)) - ) { - return undefined; - } - const counters = parseCounters(value.counters); - if (!counters) return undefined; - const activity = parseActivity(value.activity); - if (value.activity !== undefined && !activity) return undefined; - const activation = parseActivation(value.activation); - if (value.activation !== undefined && !activation) return undefined; - const timings = parseTimings(value.timings); - if (value.timings !== undefined && !timings) return undefined; - const materialization = parseMaterialization(value.materialization); - if (value.materialization !== undefined && !materialization) return undefined; - const ownerStart = value.owner.processStartIdentity; - if (ownerStart !== undefined && !isText(ownerStart, 256)) return undefined; - const subphase = value.subphase; - if (subphase !== undefined && !isText(subphase, 64)) return undefined; - const error = parseError(value.error); - if (value.error !== undefined && !error) return undefined; - const eta = parseEta(value.eta); - if (value.eta !== undefined && !eta) return undefined; - const extraction = parseExtraction(value.extraction); - if (value.extraction !== undefined && !extraction) return undefined; - const result = parseResult(value.result); - if (value.result !== undefined && !result) return undefined; - const request = parseRequest(value.request); - if (value.request !== undefined && !request) return undefined; - const resolution = parseResolution(value.resolution); - if (value.resolution !== undefined && !resolution) return undefined; - const displayName = value.identity.displayName; - if (displayName !== undefined && !isText(displayName, 256)) return undefined; - return { - ...(activation ? {activation} : {}), - ...(activity ? {activity} : {}), - buildId: value.buildId, - counters, - ...(error ? {error} : {}), - ...(eta ? {eta} : {}), - ...(extraction ? {extraction} : {}), - identity: { - checkoutId: value.identity.checkoutId, - commit: value.identity.commit, - ...(displayName ? {displayName} : {}), - repositoryId: value.identity.repositoryId, - worktreeId: value.identity.worktreeId, - }, - ...(materialization ? {materialization} : {}), - owner: { - processId: Number(value.owner.processId), - ...(ownerStart ? {processStartIdentity: ownerStart} : {}), - runtime: 'bun', - runtimeVersion: value.owner.runtimeVersion, - }, - phase: value.phase as CodeGraphProgress['phase'], - ...(request ? {request} : {}), - ...(resolution ? {resolution} : {}), - ...(result ? {result} : {}), - schemaVersion: CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION, - state: value.state as CodeGraphBuildState, - ...(subphase ? {subphase} : {}), - ...(timings ? {timings} : {}), - timestamps: { - ...(timestamps.completedAt ? {completedAt: timestamps.completedAt} : {}), - heartbeatAt: timestamps.heartbeatAt, - lastProgressAt: timestamps.lastProgressAt, - phaseStartedAt: timestamps.phaseStartedAt, - startedAt: timestamps.startedAt, - updatedAt: timestamps.updatedAt, - }, - }; -} - -function parseActivation(value: unknown): CodeGraphBuildActivation | undefined { - if (!isRecord(value)) return undefined; - const activity = parseActivationActivity(value.activity); - return activity ? {activity} : undefined; -} - -function parseActivationActivity(value: unknown): CodeGraphBuildActivation['activity'] | undefined { - if ( - !isRecord(value) || - ![ - 'checkpointing-snapshot', - 'committing-snapshot', - 'copying-edges', - 'copying-files', - 'copying-lookup-keys', - 'copying-reexports', - 'copying-symbols', - 'copying-terms', - 'copying-workspace', - 'recording-completion', - 'validating-input', - ].includes(String(value.stage)) || - !['completed', 'progress', 'started'].includes(String(value.state)) || - !isNonNegativeFinite(value.elapsedMilliseconds) || - !isNonNegativeFinite(value.stageElapsedMilliseconds) || - !isTimestamp(value.startedAt) - ) { - return undefined; - } - if (value.rows !== undefined && !isNonNegativeSafeInteger(value.rows)) return undefined; - if (value.transactionMilliseconds !== undefined && !isNonNegativeFinite(value.transactionMilliseconds)) { - return undefined; - } - return { - elapsedMilliseconds: Number(value.elapsedMilliseconds), - ...(value.rows === undefined ? {} : {rows: Number(value.rows)}), - stage: value.stage as CodeGraphActivationActivity['stage'], - stageElapsedMilliseconds: Number(value.stageElapsedMilliseconds), - startedAt: value.startedAt, - state: value.state as CodeGraphActivationActivity['state'], - ...(value.transactionMilliseconds === undefined - ? {} - : {transactionMilliseconds: Number(value.transactionMilliseconds)}), - }; -} - -function parseResolution(value: unknown): CodeGraphBuildResolution | undefined { - if (!isRecord(value)) return undefined; - const activity = parseResolutionActivity(value.activity); - return activity ? {activity} : undefined; -} - -function parseResolutionActivity(value: unknown): CodeGraphBuildResolution['activity'] | undefined { - if (!isRecord(value) || !isTimestamp(value.startedAt)) return undefined; - for (const key of [ - 'aliasesDiscovered', - 'pageCompleted', - 'pageTotal', - 'pagesCompleted', - 'pass', - 'referencesCompleted', - 'referencesExamined', - 'referencesTotal', - 'resolved', - ] as const) { - if (!isNonNegativeSafeInteger(value[key])) return undefined; - } - for (const key of ['elapsedMilliseconds', 'matchingMilliseconds', 'transactionMilliseconds'] as const) { - if (!isNonNegativeFinite(value[key])) return undefined; - } - if ( - Number(value.pass) < 1 || - Number(value.pageCompleted) > Number(value.pageTotal) || - Number(value.referencesCompleted) > Number(value.referencesTotal) || - Number(value.pageCompleted) > Number(value.pagesCompleted) || - Number(value.resolved) > Number(value.referencesExamined) - ) { - return undefined; - } - return { - aliasesDiscovered: Number(value.aliasesDiscovered), - elapsedMilliseconds: Number(value.elapsedMilliseconds), - matchingMilliseconds: Number(value.matchingMilliseconds), - pageCompleted: Number(value.pageCompleted), - pageTotal: Number(value.pageTotal), - pagesCompleted: Number(value.pagesCompleted), - pass: Number(value.pass), - referencesCompleted: Number(value.referencesCompleted), - referencesExamined: Number(value.referencesExamined), - referencesTotal: Number(value.referencesTotal), - resolved: Number(value.resolved), - startedAt: value.startedAt, - transactionMilliseconds: Number(value.transactionMilliseconds), - }; -} - -function parseMaterialization(value: unknown): CodeGraphBuildMaterialization | undefined { - if (!isRecord(value)) return undefined; - const activity = parseMaterializationActivity(value.activity); - if (value.activity !== undefined && !activity) return undefined; - const metrics = parseMaterializationMetrics(value.metrics); - if (value.metrics !== undefined && !metrics) return undefined; - if (!activity && !metrics) return undefined; - return {...(activity ? {activity} : {}), ...(metrics ? {metrics} : {})}; -} - -function parseMaterializationActivity(value: unknown): CodeGraphBuildMaterialization['activity'] | undefined { - if ( - !isRecord(value) || - !isBatchProgress(value.batchCompleted, value.batchTotal) || - !Number.isSafeInteger(value.sourceBytes) || - Number(value.sourceBytes) < 0 || - ![ - 'attributing', - 'committing', - 'loading-cache', - 'preparing-rows', - 'writing-analysis', - 'writing-candidates', - 'writing-edges', - 'writing-facts', - 'writing-lookups', - 'writing-references', - 'writing-receipt', - 'writing-symbols', - 'writing-terms', - ].includes(String(value.stage)) || - !isTimestamp(value.startedAt) - ) { - return undefined; - } - if (value.cachedFactBytes !== undefined && !isNonNegativeSafeInteger(value.cachedFactBytes)) return undefined; - if (value.elapsedMilliseconds !== undefined && !isNonNegativeFinite(value.elapsedMilliseconds)) return undefined; - if (value.factsBytes !== undefined && !isNonNegativeSafeInteger(value.factsBytes)) return undefined; - if (value.stageElapsedMilliseconds !== undefined && !isNonNegativeFinite(value.stageElapsedMilliseconds)) { - return undefined; - } - if (value.transactionMilliseconds !== undefined && !isNonNegativeFinite(value.transactionMilliseconds)) { - return undefined; - } - const rows = parseMaterializationRows(value.rows); - if (value.rows !== undefined && !rows) return undefined; - return { - batchCompleted: Number(value.batchCompleted), - batchTotal: Number(value.batchTotal), - ...(value.cachedFactBytes === undefined ? {} : {cachedFactBytes: Number(value.cachedFactBytes)}), - ...(value.elapsedMilliseconds === undefined ? {} : {elapsedMilliseconds: Number(value.elapsedMilliseconds)}), - ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), - ...(rows ? {rows} : {}), - sourceBytes: Number(value.sourceBytes), - stage: value.stage as CodeGraphMaterializationActivity['stage'], - ...(value.stageElapsedMilliseconds === undefined - ? {} - : {stageElapsedMilliseconds: Number(value.stageElapsedMilliseconds)}), - startedAt: value.startedAt, - ...(value.transactionMilliseconds === undefined - ? {} - : {transactionMilliseconds: Number(value.transactionMilliseconds)}), - }; -} - -function parseMaterializationMetrics(value: unknown): CodeGraphMaterializationMetrics | undefined { - if ( - !isRecord(value) || - !isBatchProgress(value.batchesCompleted, value.batchesTotal) || - !isNonNegativeSafeInteger(value.sourceBytesCompleted) || - !isNonNegativeSafeInteger(value.sourceBytesTotal) || - Number(value.sourceBytesCompleted) > Number(value.sourceBytesTotal) - ) { - return undefined; - } - if ( - value.fallbackReason !== undefined && - !VALID_MATERIALIZATION_FALLBACK_REASONS.has(value.fallbackReason as CodeGraphOverlayFallbackReason) - ) { - return undefined; - } - if (value.mode !== undefined && !['full', 'incremental-clean', 'incremental-overlay'].includes(String(value.mode))) { - return undefined; - } - for (const key of [ - 'cachedFactBytesCompleted', - 'cachedFactBytesTotal', - 'factsBytesCompleted', - 'factsBytesTotal', - ] as const) { - if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; - } - if ( - value.cachedFactBytesCompleted !== undefined && - value.cachedFactBytesTotal !== undefined && - Number(value.cachedFactBytesCompleted) > Number(value.cachedFactBytesTotal) - ) { - return undefined; - } - if ( - value.factsBytesCompleted !== undefined && - value.factsBytesTotal !== undefined && - Number(value.factsBytesCompleted) > Number(value.factsBytesTotal) - ) { - return undefined; - } - for (const key of ['attributionMilliseconds', 'loadingMilliseconds', 'transactionMilliseconds'] as const) { - if (value[key] !== undefined && !isNonNegativeFinite(value[key])) return undefined; - } - const rows = parseMaterializationRows(value.rows); - if (value.rows !== undefined && !rows) return undefined; - const stageMilliseconds = parseMaterializationStageMilliseconds(value.stageMilliseconds); - if (value.stageMilliseconds !== undefined && !stageMilliseconds) return undefined; - const storage = parseMaterializationStorage(value.storage); - if (value.storage !== undefined && !storage) return undefined; - if (storage?.estimateBasis === 'cached-fact-bytes' && value.cachedFactBytesTotal === undefined) return undefined; - if (storage?.estimateBasis === 'final-fact-bytes' && value.factsBytesTotal === undefined) return undefined; - return { - ...(value.fallbackReason === undefined - ? {} - : {fallbackReason: value.fallbackReason as CodeGraphMaterializationMetrics['fallbackReason']}), - ...(value.attributionMilliseconds === undefined - ? {} - : {attributionMilliseconds: Number(value.attributionMilliseconds)}), - batchesCompleted: Number(value.batchesCompleted), - batchesTotal: Number(value.batchesTotal), - ...(value.cachedFactBytesCompleted === undefined - ? {} - : {cachedFactBytesCompleted: Number(value.cachedFactBytesCompleted)}), - ...(value.cachedFactBytesTotal === undefined ? {} : {cachedFactBytesTotal: Number(value.cachedFactBytesTotal)}), - ...(value.factsBytesCompleted === undefined ? {} : {factsBytesCompleted: Number(value.factsBytesCompleted)}), - ...(value.factsBytesTotal === undefined ? {} : {factsBytesTotal: Number(value.factsBytesTotal)}), - ...(value.loadingMilliseconds === undefined ? {} : {loadingMilliseconds: Number(value.loadingMilliseconds)}), - ...(value.mode === undefined ? {} : {mode: value.mode as CodeGraphMaterializationMetrics['mode']}), - ...(rows ? {rows} : {}), - sourceBytesCompleted: Number(value.sourceBytesCompleted), - sourceBytesTotal: Number(value.sourceBytesTotal), - ...(stageMilliseconds ? {stageMilliseconds} : {}), - ...(storage ? {storage} : {}), - ...(value.transactionMilliseconds === undefined - ? {} - : {transactionMilliseconds: Number(value.transactionMilliseconds)}), - }; -} - -function parseMaterializationStageMilliseconds( - value: unknown, -): CodeGraphMaterializationMetrics['stageMilliseconds'] | undefined { - if (!isRecord(value)) return undefined; - const stages = [ - 'attributing', - 'committing', - 'loading-cache', - 'preparing-rows', - 'writing-analysis', - 'writing-candidates', - 'writing-edges', - 'writing-facts', - 'writing-lookups', - 'writing-receipt', - 'writing-references', - 'writing-symbols', - 'writing-terms', - ] as const satisfies readonly CodeGraphMaterializationActivity['stage'][]; - const allowed = new Set(stages); - const parsed: Partial> = {}; - for (const [stage, milliseconds] of Object.entries(value)) { - if (!allowed.has(stage) || !isNonNegativeFinite(milliseconds)) return undefined; - parsed[stage as CodeGraphMaterializationActivity['stage']] = Number(milliseconds); - } - return parsed; -} - -function parseMaterializationStorage( - value: unknown, -): NonNullable | undefined { - if ( - !isRecord(value) || - !isNonNegativeSafeInteger(value.temporaryDatabaseBytes) || - !isNonNegativeSafeInteger(value.temporaryDatabaseHighWaterBytes) || - Number(value.temporaryDatabaseBytes) > Number(value.temporaryDatabaseHighWaterBytes) - ) { - return undefined; - } - if ( - value.estimateBasis !== undefined && - !['cached-fact-bytes', 'final-fact-bytes', 'source-bytes-fallback'].includes(String(value.estimateBasis)) - ) { - return undefined; - } - for (const key of [ - 'availableBytes', - 'durableAvailableBytes', - 'durableDatabaseBytes', - 'durableDatabaseFileBytes', - 'durableDatabaseFileHighWaterBytes', - 'durableDatabaseGrowthBytes', - 'durableDatabaseGrowthHighWaterBytes', - 'durableDatabaseHighWaterBytes', - 'durableDatabaseStartBytes', - 'durableFilesystemBytes', - 'durableFilesystemHighWaterBytes', - 'durableJournalBytes', - 'durableJournalHighWaterBytes', - 'durableSharedMemoryBytes', - 'durableSharedMemoryHighWaterBytes', - 'durableWalBytes', - 'durableWalHighWaterBytes', - 'estimatedConcurrentBuildBytes', - 'estimatedDurableFilesystemRequiredBytes', - 'estimatedDurableSnapshotBytes', - 'estimatedJournalBytes', - 'estimatedRequiredBytes', - 'estimatedTemporaryFilesystemRequiredBytes', - 'estimatedTemporaryDatabaseBytes', - 'temporaryAvailableBytes', - ] as const) { - if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; - } - if (value.filesystemsShared !== undefined && typeof value.filesystemsShared !== 'boolean') return undefined; - if ( - value.materializationMode !== undefined && - !['direct-persistent', 'temporary-staged'].includes(String(value.materializationMode)) - ) { - return undefined; - } - for (const [current, highWater] of [ - ['durableDatabaseFileBytes', 'durableDatabaseFileHighWaterBytes'], - ['durableDatabaseGrowthBytes', 'durableDatabaseGrowthHighWaterBytes'], - ['durableFilesystemBytes', 'durableFilesystemHighWaterBytes'], - ['durableJournalBytes', 'durableJournalHighWaterBytes'], - ['durableSharedMemoryBytes', 'durableSharedMemoryHighWaterBytes'], - ['durableWalBytes', 'durableWalHighWaterBytes'], - ] as const) { - if ( - value[current] !== undefined && - value[highWater] !== undefined && - Number(value[current]) > Number(value[highWater]) - ) { - return undefined; - } - } - if ( - value.durableDatabaseBytes !== undefined && - value.durableDatabaseHighWaterBytes !== undefined && - Number(value.durableDatabaseBytes) > Number(value.durableDatabaseHighWaterBytes) - ) { - return undefined; - } - if ( - value.estimatedRequiredBytes !== undefined && - value.estimatedConcurrentBuildBytes !== undefined && - Number(value.estimatedRequiredBytes) < Number(value.estimatedConcurrentBuildBytes) - ) { - return undefined; - } - return { - ...(value.availableBytes === undefined ? {} : {availableBytes: Number(value.availableBytes)}), - ...(value.durableAvailableBytes === undefined ? {} : {durableAvailableBytes: Number(value.durableAvailableBytes)}), - ...(value.durableDatabaseBytes === undefined ? {} : {durableDatabaseBytes: Number(value.durableDatabaseBytes)}), - ...(value.durableDatabaseFileBytes === undefined - ? {} - : {durableDatabaseFileBytes: Number(value.durableDatabaseFileBytes)}), - ...(value.durableDatabaseFileHighWaterBytes === undefined - ? {} - : {durableDatabaseFileHighWaterBytes: Number(value.durableDatabaseFileHighWaterBytes)}), - ...(value.durableDatabaseGrowthBytes === undefined - ? {} - : {durableDatabaseGrowthBytes: Number(value.durableDatabaseGrowthBytes)}), - ...(value.durableDatabaseGrowthHighWaterBytes === undefined - ? {} - : {durableDatabaseGrowthHighWaterBytes: Number(value.durableDatabaseGrowthHighWaterBytes)}), - ...(value.durableDatabaseHighWaterBytes === undefined - ? {} - : {durableDatabaseHighWaterBytes: Number(value.durableDatabaseHighWaterBytes)}), - ...(value.durableDatabaseStartBytes === undefined - ? {} - : {durableDatabaseStartBytes: Number(value.durableDatabaseStartBytes)}), - ...(value.durableFilesystemBytes === undefined - ? {} - : {durableFilesystemBytes: Number(value.durableFilesystemBytes)}), - ...(value.durableFilesystemHighWaterBytes === undefined - ? {} - : {durableFilesystemHighWaterBytes: Number(value.durableFilesystemHighWaterBytes)}), - ...(value.durableJournalBytes === undefined ? {} : {durableJournalBytes: Number(value.durableJournalBytes)}), - ...(value.durableJournalHighWaterBytes === undefined - ? {} - : {durableJournalHighWaterBytes: Number(value.durableJournalHighWaterBytes)}), - ...(value.durableSharedMemoryBytes === undefined - ? {} - : {durableSharedMemoryBytes: Number(value.durableSharedMemoryBytes)}), - ...(value.durableSharedMemoryHighWaterBytes === undefined - ? {} - : {durableSharedMemoryHighWaterBytes: Number(value.durableSharedMemoryHighWaterBytes)}), - ...(value.durableWalBytes === undefined ? {} : {durableWalBytes: Number(value.durableWalBytes)}), - ...(value.durableWalHighWaterBytes === undefined - ? {} - : {durableWalHighWaterBytes: Number(value.durableWalHighWaterBytes)}), - ...(value.estimateBasis === undefined - ? {} - : { - estimateBasis: value.estimateBasis as 'cached-fact-bytes' | 'final-fact-bytes' | 'source-bytes-fallback', - }), - ...(value.estimatedConcurrentBuildBytes === undefined - ? {} - : {estimatedConcurrentBuildBytes: Number(value.estimatedConcurrentBuildBytes)}), - ...(value.estimatedDurableFilesystemRequiredBytes === undefined - ? {} - : {estimatedDurableFilesystemRequiredBytes: Number(value.estimatedDurableFilesystemRequiredBytes)}), - ...(value.estimatedDurableSnapshotBytes === undefined - ? {} - : {estimatedDurableSnapshotBytes: Number(value.estimatedDurableSnapshotBytes)}), - ...(value.estimatedJournalBytes === undefined ? {} : {estimatedJournalBytes: Number(value.estimatedJournalBytes)}), - ...(value.estimatedRequiredBytes === undefined - ? {} - : {estimatedRequiredBytes: Number(value.estimatedRequiredBytes)}), - ...(value.estimatedTemporaryFilesystemRequiredBytes === undefined - ? {} - : {estimatedTemporaryFilesystemRequiredBytes: Number(value.estimatedTemporaryFilesystemRequiredBytes)}), - ...(value.estimatedTemporaryDatabaseBytes === undefined - ? {} - : {estimatedTemporaryDatabaseBytes: Number(value.estimatedTemporaryDatabaseBytes)}), - ...(value.filesystemsShared === undefined ? {} : {filesystemsShared: value.filesystemsShared}), - ...(value.materializationMode === undefined - ? {} - : {materializationMode: value.materializationMode as 'direct-persistent' | 'temporary-staged'}), - ...(value.temporaryAvailableBytes === undefined - ? {} - : {temporaryAvailableBytes: Number(value.temporaryAvailableBytes)}), - temporaryDatabaseBytes: Number(value.temporaryDatabaseBytes), - temporaryDatabaseHighWaterBytes: Number(value.temporaryDatabaseHighWaterBytes), - }; -} - -function parseMaterializationRows(value: unknown): CodeGraphMaterializationRows | undefined { - if (!isRecord(value)) return undefined; - const keys = [ - 'deduplicatedEdges', - 'deduplicatedReferences', - 'edges', - 'lookupKeys', - 'referenceCandidates', - 'references', - 'reexports', - 'symbols', - 'terms', - ] as const; - for (const key of keys) { - if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; - } - return Object.fromEntries(keys.flatMap(key => (value[key] === undefined ? [] : [[key, Number(value[key])]]))); -} - -function isBatchProgress(completed: unknown, total: unknown): boolean { - return isNonNegativeSafeInteger(completed) && isNonNegativeSafeInteger(total) && Number(completed) <= Number(total); -} - -function isNonNegativeSafeInteger(value: unknown): boolean { - return Number.isSafeInteger(value) && Number(value) >= 0; -} - -function parseActivity(value: unknown): CodeGraphBuildActivity | undefined { - if ( - !isRecord(value) || - !Number.isSafeInteger(value.batchCompleted) || - !Number.isSafeInteger(value.batchTotal) || - Number(value.batchCompleted) < 0 || - Number(value.batchTotal) < 0 || - Number(value.batchCompleted) > Number(value.batchTotal) || - !Number.isSafeInteger(value.bytes) || - Number(value.bytes) < 0 || - !isText(value.language, 64) || - !['extracting', 'persisting', 'reading'].includes(String(value.stage)) || - (value.classifier !== undefined && !isText(value.classifier, 64)) || - (value.degraded !== undefined && typeof value.degraded !== 'boolean') || - (value.role !== undefined && !isText(value.role, 64)) || - (value.sizeBucket !== undefined && !isCodeGraphSourceSizeBucket(value.sizeBucket)) - ) { - return undefined; - } - for (const key of ['factsBytes', 'relations', 'symbols'] as const) { - if (value[key] !== undefined && (!Number.isSafeInteger(value[key]) || Number(value[key]) < 0)) return undefined; - } - for (const key of ['parseMilliseconds', 'persistMilliseconds'] as const) { - if (value[key] !== undefined && !isNonNegativeFinite(value[key])) return undefined; - } - return { - batchCompleted: Number(value.batchCompleted), - batchTotal: Number(value.batchTotal), - bytes: Number(value.bytes), - ...(value.classifier === undefined ? {} : {classifier: value.classifier}), - ...(typeof value.degraded === 'boolean' ? {degraded: value.degraded} : {}), - ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), - language: value.language, - ...(value.parseMilliseconds === undefined ? {} : {parseMilliseconds: Number(value.parseMilliseconds)}), - ...(value.persistMilliseconds === undefined ? {} : {persistMilliseconds: Number(value.persistMilliseconds)}), - ...(value.relations === undefined ? {} : {relations: Number(value.relations)}), - ...(value.role === undefined ? {} : {role: value.role}), - ...(value.sizeBucket === undefined ? {} : {sizeBucket: value.sizeBucket}), - stage: value.stage as CodeGraphBuildActivity['stage'], - ...(value.symbols === undefined ? {} : {symbols: Number(value.symbols)}), - }; -} - -function parseExtraction(value: unknown): CodeGraphBuildExtraction | undefined { - if ( - !isRecord(value) || - !isNonNegativeSafeInteger(value.completedFiles) || - !isNonNegativeSafeInteger(value.slowFiles) || - Number(value.slowFiles) > Number(value.completedFiles) || - !Array.isArray(value.topSlowFiles) || - value.topSlowFiles.length > CODE_GRAPH_TOP_SLOW_FILE_LIMIT - ) { - return undefined; - } - const topSlowFiles = value.topSlowFiles.map(parseSlowFileTelemetry); - if (topSlowFiles.some(sample => sample === undefined)) return undefined; - const metrics = value.metrics === undefined ? undefined : parseScanningMetrics(value.metrics); - if (value.metrics !== undefined && metrics === undefined) return undefined; - const samples = topSlowFiles as CodeGraphSlowFileTelemetry[]; - if ( - samples.some( - (sample, index) => - index > 0 && - (sample.durationMilliseconds > samples[index - 1]!.durationMilliseconds || - (sample.durationMilliseconds === samples[index - 1]!.durationMilliseconds && - sample.pathHash.localeCompare(samples[index - 1]!.pathHash) < 0)), - ) - ) { - return undefined; - } - return { - completedFiles: Number(value.completedFiles), - ...(metrics === undefined ? {} : {metrics}), - slowFiles: Number(value.slowFiles), - topSlowFiles: samples, - }; -} - -function parseScanningMetrics(value: unknown): CodeGraphScanningMetrics | undefined { - if (!isRecord(value)) return undefined; - for (const key of [ - 'factsBytesCompleted', - 'sourceBytesCompleted', - 'sourceBytesTotal', - 'workUnitsCompleted', - 'workUnitsTotal', - ] as const) { - if (!isNonNegativeSafeInteger(value[key])) return undefined; - } - if ( - Number(value.sourceBytesCompleted) > Number(value.sourceBytesTotal) || - Number(value.workUnitsCompleted) > Number(value.workUnitsTotal) - ) { - return undefined; - } - return { - factsBytesCompleted: Number(value.factsBytesCompleted), - sourceBytesCompleted: Number(value.sourceBytesCompleted), - sourceBytesTotal: Number(value.sourceBytesTotal), - workUnitsCompleted: Number(value.workUnitsCompleted), - workUnitsTotal: Number(value.workUnitsTotal), - }; -} - -function parseSlowFileTelemetry(value: unknown): CodeGraphSlowFileTelemetry | undefined { - if ( - !isRecord(value) || - !isText(value.classifier, 64) || - !isNonNegativeFinite(value.durationMilliseconds) || - !isText(value.extension, 16) || - !isText(value.language, 64) || - typeof value.pathHash !== 'string' || - !/^[a-f0-9]{64}$/.test(value.pathHash) || - !isText(value.role, 64) || - !isCodeGraphSourceSizeBucket(value.sizeBucket) || - !isNonNegativeSafeInteger(value.sourceBytes) || - (value.degraded !== undefined && typeof value.degraded !== 'boolean') - ) { - return undefined; - } - for (const key of ['factsBytes', 'relations', 'symbols'] as const) { - if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; - } - return { - classifier: value.classifier, - ...(value.degraded === undefined ? {} : {degraded: value.degraded}), - durationMilliseconds: Number(value.durationMilliseconds), - extension: value.extension, - ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), - language: value.language, - pathHash: value.pathHash, - ...(value.relations === undefined ? {} : {relations: Number(value.relations)}), - role: value.role, - sizeBucket: value.sizeBucket, - sourceBytes: Number(value.sourceBytes), - ...(value.symbols === undefined ? {} : {symbols: Number(value.symbols)}), - }; -} - -function parseTimings(value: unknown): CodeGraphBuildTimings | undefined { - return isRecord(value) && - isNonNegativeFinite(value.extractionMilliseconds) && - isNonNegativeFinite(value.persistenceMilliseconds) && - isNonNegativeFinite(value.readingMilliseconds) - ? { - extractionMilliseconds: Number(value.extractionMilliseconds), - persistenceMilliseconds: Number(value.persistenceMilliseconds), - readingMilliseconds: Number(value.readingMilliseconds), - } - : undefined; -} - -function isNonNegativeFinite(value: unknown): boolean { - return typeof value === 'number' && Number.isFinite(value) && value >= 0; -} - -function parseRequest(value: unknown): CodeGraphBuildStatus['request'] | undefined { - return isRecord(value) && typeof value.key === 'string' && HASH_ID.test(value.key) ? {key: value.key} : undefined; -} - -function parseCounters(value: unknown): CodeGraphBuildCounters | undefined { - if (!isRecord(value)) return undefined; - const keys = [ - 'accepted', - 'completed', - 'edges', - 'embedded', - 'excluded', - 'pagesCompleted', - 'reused', - 'resolved', - 'rowsDeleted', - 'skipped', - 'symbols', - 'total', - ] as const; - for (const key of keys) { - const counter = value[key]; - if (counter !== undefined && (!Number.isSafeInteger(counter) || Number(counter) < 0)) return undefined; - } - if (value.unit !== undefined && !['files', 'references', 'snapshots', 'symbols'].includes(String(value.unit))) - return undefined; - return Object.fromEntries( - [...keys, 'unit' as const].flatMap(key => (value[key] === undefined ? [] : [[key, value[key]]])), - ) as CodeGraphBuildCounters; -} - -function parseError(value: unknown): CodeGraphBuildStatus['error'] | undefined { - return isRecord(value) && isText(value.summary, 300) ? {summary: value.summary} : undefined; -} - -function parseEta(value: unknown): CodeGraphBuildStatus['eta'] | undefined { - return isRecord(value) && - value.scope === 'phase' && - ['high', 'low', 'medium'].includes(String(value.confidence)) && - (value.basis === undefined || - ['cached-fact-bytes', 'extraction-work', 'files', 'final-fact-bytes', 'source-bytes'].includes( - String(value.basis), - )) && - Number.isSafeInteger(value.remainingMilliseconds) && - Number(value.remainingMilliseconds) >= 0 - ? { - ...(value.basis === undefined - ? {} - : { - basis: value.basis as - 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes', - }), - confidence: value.confidence as 'high' | 'low' | 'medium', - remainingMilliseconds: Number(value.remainingMilliseconds), - scope: 'phase', - } - : undefined; -} - -function parseResult(value: unknown): CodeGraphBuildStatus['result'] | undefined { - if (!isRecord(value) || typeof value.dirty !== 'boolean' || !isText(value.snapshotId, 128)) return undefined; - for (const key of ['edges', 'files', 'symbols'] as const) { - if (!Number.isSafeInteger(value[key]) || Number(value[key]) < 0) return undefined; - } - return { - dirty: value.dirty, - edges: Number(value.edges), - files: Number(value.files), - snapshotId: value.snapshotId, - symbols: Number(value.symbols), - }; -} - function pruneCodeGraphBuildHistory( fs: FileSystem.FileSystem, path: Path.Path, @@ -2045,7 +1251,9 @@ interface BuildHistoryDirectoryAuthority { type BuildHistoryCursor = {readonly mode: 'reset'} | {readonly afterBuildId: string; readonly mode: 'scan'}; -class InvalidBuildHistorySidecarError extends Error {} +class InvalidBuildHistorySidecarError extends Error { + readonly _tag = 'InvalidBuildHistorySidecarError' as const; +} const pruneCodeGraphBuildHistoryUnitWithServices = Effect.fn('codeGraph.buildStatus.pruneHistoryUnitUnsafe')(function* ( fs: FileSystem.FileSystem, @@ -2322,7 +1530,8 @@ const readBuildHistoryManagerContext = Effect.fn('codeGraph.buildStatus.readHist isRecord(value) && value.schemaVersion === MANAGER_CONTEXT_SCHEMA_VERSION && value.buildId === buildId && - isText(value.worktreePath, 4_096) + isText(value.worktreePath, 4_096) && + (value.branch === undefined || isText(value.branch, 1_024)) ); }, catch: () => new InvalidBuildHistorySidecarError('Build history Manager context is invalid JSON.'), @@ -2739,19 +1948,3 @@ function privacySafeError(cause: unknown): string { function boundedText(value: string, maximum: number): string { return value.length <= maximum ? value : `${value.slice(0, Math.max(0, maximum - 1))}…`; } - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function isHash(value: unknown): value is string { - return typeof value === 'string' && HASH_ID.test(value); -} - -function isText(value: unknown, maximum: number): value is string { - return typeof value === 'string' && value.length > 0 && value.length <= maximum && !/[\p{Cc}]/u.test(value); -} - -function isTimestamp(value: unknown): value is string { - return isText(value, 64) && Number.isFinite(Date.parse(value)); -} diff --git a/src/code_graph/build_status_codec.ts b/src/code_graph/build_status_codec.ts new file mode 100644 index 00000000..389cf1d0 --- /dev/null +++ b/src/code_graph/build_status_codec.ts @@ -0,0 +1,856 @@ +import { + CODE_GRAPH_BUILD_COMMIT_ID, + CODE_GRAPH_BUILD_ID, + CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION, + isBuildStatusHash, + isBuildStatusRecord, + isBuildStatusText, + isBuildStatusTimestamp, +} from './build_status_validation.js'; +import { + CODE_GRAPH_TOP_SLOW_FILE_LIMIT, + isCodeGraphSourceSizeBucket, + type CodeGraphScanningMetrics, + type CodeGraphSlowFileTelemetry, +} from './progress_telemetry.js'; +import type { + CodeGraphActivationActivity, + CodeGraphMaterializationActivity, + CodeGraphMaterializationMetrics, + CodeGraphMaterializationRows, + CodeGraphOverlayFallbackReason, + CodeGraphProgress, +} from './types.js'; +import type { + CodeGraphBuildActivation, + CodeGraphBuildActivity, + CodeGraphBuildCounters, + CodeGraphBuildExtraction, + CodeGraphBuildMaterialization, + CodeGraphBuildResolution, + CodeGraphBuildState, + CodeGraphBuildStatus, + CodeGraphBuildTimings, +} from './build_status.js'; + +const BUILD_ID = CODE_GRAPH_BUILD_ID; +const COMMIT_ID = CODE_GRAPH_BUILD_COMMIT_ID; +const VALID_PHASES = new Set([ + 'activating', + 'embedding', + 'materializing', + 'reclaiming', + 'registering', + 'resolving', + 'scanning', + 'waiting', +]); +const VALID_STATES = new Set(['completed', 'failed', 'queued', 'running']); +const VALID_MATERIALIZATION_FALLBACK_REASONS = new Set([ + 'cache-incomplete', + 'disabled', + 'dynamic-aliases', + 'extractor-context-changed', + 'fact-budget-expanded', + 'file-set-changed', + 'forced-full-rebuild', + 'incremental-rewrite-unbounded', + 'no-materialized-changes', + 'project-closure-incomplete', + 'project-closure-unbounded', + 'reexport-closure-unbounded', + 'resolution-surface-changed', + 'staging-identity-mismatch', + 'staging-unavailable', + 'workspace-changed', +]); + +const isRecord = isBuildStatusRecord; +const isHash = isBuildStatusHash; +const isText = isBuildStatusText; +const isTimestamp = isBuildStatusTimestamp; + +export function parseCodeGraphBuildStatus(value: unknown): CodeGraphBuildStatus | undefined { + if (!isRecord(value) || value.schemaVersion !== CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION) return undefined; + if (!isText(value.buildId, 64) || !BUILD_ID.test(value.buildId)) return undefined; + if (!isRecord(value.identity) || !isRecord(value.owner) || !isRecord(value.timestamps)) return undefined; + if ( + !isHash(value.identity.repositoryId) || + !isHash(value.identity.checkoutId) || + !isHash(value.identity.worktreeId) || + !isText(value.identity.commit, 64) || + !COMMIT_ID.test(value.identity.commit) || + !Number.isSafeInteger(value.owner.processId) || + Number(value.owner.processId) <= 0 || + value.owner.runtime !== 'bun' || + !isText(value.owner.runtimeVersion, 64) || + !VALID_PHASES.has(value.phase as CodeGraphProgress['phase']) || + !VALID_STATES.has(value.state as CodeGraphBuildState) + ) { + return undefined; + } + const timestamps = value.timestamps; + if ( + !isTimestamp(timestamps.startedAt) || + !isTimestamp(timestamps.phaseStartedAt) || + !isTimestamp(timestamps.lastProgressAt) || + !isTimestamp(timestamps.heartbeatAt) || + !isTimestamp(timestamps.updatedAt) || + (timestamps.completedAt !== undefined && !isTimestamp(timestamps.completedAt)) + ) { + return undefined; + } + const counters = parseCounters(value.counters); + if (!counters) return undefined; + const activity = parseActivity(value.activity); + if (value.activity !== undefined && !activity) return undefined; + const activation = parseActivation(value.activation); + if (value.activation !== undefined && !activation) return undefined; + const timings = parseTimings(value.timings); + if (value.timings !== undefined && !timings) return undefined; + const materialization = parseMaterialization(value.materialization); + if (value.materialization !== undefined && !materialization) return undefined; + const ownerStart = value.owner.processStartIdentity; + if (ownerStart !== undefined && !isText(ownerStart, 256)) return undefined; + const subphase = value.subphase; + if (subphase !== undefined && !isText(subphase, 64)) return undefined; + const error = parseError(value.error); + if (value.error !== undefined && !error) return undefined; + const eta = parseEta(value.eta); + if (value.eta !== undefined && !eta) return undefined; + const extraction = parseExtraction(value.extraction); + if (value.extraction !== undefined && !extraction) return undefined; + const result = parseResult(value.result); + if (value.result !== undefined && !result) return undefined; + const request = parseRequest(value.request); + if (value.request !== undefined && !request) return undefined; + const resolution = parseResolution(value.resolution); + if (value.resolution !== undefined && !resolution) return undefined; + const displayName = value.identity.displayName; + if (displayName !== undefined && !isText(displayName, 256)) return undefined; + return { + ...(activation ? {activation} : {}), + ...(activity ? {activity} : {}), + buildId: value.buildId, + counters, + ...(error ? {error} : {}), + ...(eta ? {eta} : {}), + ...(extraction ? {extraction} : {}), + identity: { + checkoutId: value.identity.checkoutId, + commit: value.identity.commit, + ...(displayName ? {displayName} : {}), + repositoryId: value.identity.repositoryId, + worktreeId: value.identity.worktreeId, + }, + ...(materialization ? {materialization} : {}), + owner: { + processId: Number(value.owner.processId), + ...(ownerStart ? {processStartIdentity: ownerStart} : {}), + runtime: 'bun', + runtimeVersion: value.owner.runtimeVersion, + }, + phase: value.phase as CodeGraphProgress['phase'], + ...(request ? {request} : {}), + ...(resolution ? {resolution} : {}), + ...(result ? {result} : {}), + schemaVersion: CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION, + state: value.state as CodeGraphBuildState, + ...(subphase ? {subphase} : {}), + ...(timings ? {timings} : {}), + timestamps: { + ...(timestamps.completedAt ? {completedAt: timestamps.completedAt} : {}), + heartbeatAt: timestamps.heartbeatAt, + lastProgressAt: timestamps.lastProgressAt, + phaseStartedAt: timestamps.phaseStartedAt, + startedAt: timestamps.startedAt, + updatedAt: timestamps.updatedAt, + }, + }; +} + +function parseActivation(value: unknown): CodeGraphBuildActivation | undefined { + if (!isRecord(value)) return undefined; + const activity = parseActivationActivity(value.activity); + return activity ? {activity} : undefined; +} + +function parseActivationActivity(value: unknown): CodeGraphBuildActivation['activity'] | undefined { + if ( + !isRecord(value) || + ![ + 'checkpointing-snapshot', + 'committing-snapshot', + 'copying-edges', + 'copying-files', + 'copying-lookup-keys', + 'copying-reexports', + 'copying-symbols', + 'copying-terms', + 'copying-workspace', + 'recording-completion', + 'validating-input', + ].includes(String(value.stage)) || + !['completed', 'progress', 'started'].includes(String(value.state)) || + !isNonNegativeFinite(value.elapsedMilliseconds) || + !isNonNegativeFinite(value.stageElapsedMilliseconds) || + !isTimestamp(value.startedAt) + ) { + return undefined; + } + if (value.rows !== undefined && !isNonNegativeSafeInteger(value.rows)) return undefined; + if (value.transactionMilliseconds !== undefined && !isNonNegativeFinite(value.transactionMilliseconds)) { + return undefined; + } + return { + elapsedMilliseconds: Number(value.elapsedMilliseconds), + ...(value.rows === undefined ? {} : {rows: Number(value.rows)}), + stage: value.stage as CodeGraphActivationActivity['stage'], + stageElapsedMilliseconds: Number(value.stageElapsedMilliseconds), + startedAt: value.startedAt, + state: value.state as CodeGraphActivationActivity['state'], + ...(value.transactionMilliseconds === undefined + ? {} + : {transactionMilliseconds: Number(value.transactionMilliseconds)}), + }; +} + +function parseResolution(value: unknown): CodeGraphBuildResolution | undefined { + if (!isRecord(value)) return undefined; + const activity = parseResolutionActivity(value.activity); + return activity ? {activity} : undefined; +} + +function parseResolutionActivity(value: unknown): CodeGraphBuildResolution['activity'] | undefined { + if (!isRecord(value) || !isTimestamp(value.startedAt)) return undefined; + for (const key of [ + 'aliasesDiscovered', + 'pageCompleted', + 'pageTotal', + 'pagesCompleted', + 'pass', + 'referencesCompleted', + 'referencesExamined', + 'referencesTotal', + 'resolved', + ] as const) { + if (!isNonNegativeSafeInteger(value[key])) return undefined; + } + for (const key of ['elapsedMilliseconds', 'matchingMilliseconds', 'transactionMilliseconds'] as const) { + if (!isNonNegativeFinite(value[key])) return undefined; + } + if ( + Number(value.pass) < 1 || + Number(value.pageCompleted) > Number(value.pageTotal) || + Number(value.referencesCompleted) > Number(value.referencesTotal) || + Number(value.pageCompleted) > Number(value.pagesCompleted) || + Number(value.resolved) > Number(value.referencesExamined) + ) { + return undefined; + } + return { + aliasesDiscovered: Number(value.aliasesDiscovered), + elapsedMilliseconds: Number(value.elapsedMilliseconds), + matchingMilliseconds: Number(value.matchingMilliseconds), + pageCompleted: Number(value.pageCompleted), + pageTotal: Number(value.pageTotal), + pagesCompleted: Number(value.pagesCompleted), + pass: Number(value.pass), + referencesCompleted: Number(value.referencesCompleted), + referencesExamined: Number(value.referencesExamined), + referencesTotal: Number(value.referencesTotal), + resolved: Number(value.resolved), + startedAt: value.startedAt, + transactionMilliseconds: Number(value.transactionMilliseconds), + }; +} + +function parseMaterialization(value: unknown): CodeGraphBuildMaterialization | undefined { + if (!isRecord(value)) return undefined; + const activity = parseMaterializationActivity(value.activity); + if (value.activity !== undefined && !activity) return undefined; + const metrics = parseMaterializationMetrics(value.metrics); + if (value.metrics !== undefined && !metrics) return undefined; + if (!activity && !metrics) return undefined; + return {...(activity ? {activity} : {}), ...(metrics ? {metrics} : {})}; +} + +function parseMaterializationActivity(value: unknown): CodeGraphBuildMaterialization['activity'] | undefined { + if ( + !isRecord(value) || + !isBatchProgress(value.batchCompleted, value.batchTotal) || + !Number.isSafeInteger(value.sourceBytes) || + Number(value.sourceBytes) < 0 || + ![ + 'attributing', + 'committing', + 'loading-cache', + 'preparing-rows', + 'writing-analysis', + 'writing-candidates', + 'writing-edges', + 'writing-facts', + 'writing-lookups', + 'writing-references', + 'writing-receipt', + 'writing-symbols', + 'writing-terms', + ].includes(String(value.stage)) || + !isTimestamp(value.startedAt) + ) { + return undefined; + } + if (value.cachedFactBytes !== undefined && !isNonNegativeSafeInteger(value.cachedFactBytes)) return undefined; + if (value.elapsedMilliseconds !== undefined && !isNonNegativeFinite(value.elapsedMilliseconds)) return undefined; + if (value.factsBytes !== undefined && !isNonNegativeSafeInteger(value.factsBytes)) return undefined; + if (value.stageElapsedMilliseconds !== undefined && !isNonNegativeFinite(value.stageElapsedMilliseconds)) { + return undefined; + } + if (value.transactionMilliseconds !== undefined && !isNonNegativeFinite(value.transactionMilliseconds)) { + return undefined; + } + const rows = parseMaterializationRows(value.rows); + if (value.rows !== undefined && !rows) return undefined; + return { + batchCompleted: Number(value.batchCompleted), + batchTotal: Number(value.batchTotal), + ...(value.cachedFactBytes === undefined ? {} : {cachedFactBytes: Number(value.cachedFactBytes)}), + ...(value.elapsedMilliseconds === undefined ? {} : {elapsedMilliseconds: Number(value.elapsedMilliseconds)}), + ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), + ...(rows ? {rows} : {}), + sourceBytes: Number(value.sourceBytes), + stage: value.stage as CodeGraphMaterializationActivity['stage'], + ...(value.stageElapsedMilliseconds === undefined + ? {} + : {stageElapsedMilliseconds: Number(value.stageElapsedMilliseconds)}), + startedAt: value.startedAt, + ...(value.transactionMilliseconds === undefined + ? {} + : {transactionMilliseconds: Number(value.transactionMilliseconds)}), + }; +} + +function parseMaterializationMetrics(value: unknown): CodeGraphMaterializationMetrics | undefined { + if ( + !isRecord(value) || + !isBatchProgress(value.batchesCompleted, value.batchesTotal) || + !isNonNegativeSafeInteger(value.sourceBytesCompleted) || + !isNonNegativeSafeInteger(value.sourceBytesTotal) || + Number(value.sourceBytesCompleted) > Number(value.sourceBytesTotal) + ) { + return undefined; + } + if ( + value.fallbackReason !== undefined && + !VALID_MATERIALIZATION_FALLBACK_REASONS.has(value.fallbackReason as CodeGraphOverlayFallbackReason) + ) { + return undefined; + } + if (value.mode !== undefined && !['full', 'incremental-clean', 'incremental-overlay'].includes(String(value.mode))) { + return undefined; + } + for (const key of [ + 'cachedFactBytesCompleted', + 'cachedFactBytesTotal', + 'factsBytesCompleted', + 'factsBytesTotal', + ] as const) { + if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; + } + if ( + value.cachedFactBytesCompleted !== undefined && + value.cachedFactBytesTotal !== undefined && + Number(value.cachedFactBytesCompleted) > Number(value.cachedFactBytesTotal) + ) { + return undefined; + } + if ( + value.factsBytesCompleted !== undefined && + value.factsBytesTotal !== undefined && + Number(value.factsBytesCompleted) > Number(value.factsBytesTotal) + ) { + return undefined; + } + for (const key of ['attributionMilliseconds', 'loadingMilliseconds', 'transactionMilliseconds'] as const) { + if (value[key] !== undefined && !isNonNegativeFinite(value[key])) return undefined; + } + const rows = parseMaterializationRows(value.rows); + if (value.rows !== undefined && !rows) return undefined; + const stageMilliseconds = parseMaterializationStageMilliseconds(value.stageMilliseconds); + if (value.stageMilliseconds !== undefined && !stageMilliseconds) return undefined; + const storage = parseMaterializationStorage(value.storage); + if (value.storage !== undefined && !storage) return undefined; + if (storage?.estimateBasis === 'cached-fact-bytes' && value.cachedFactBytesTotal === undefined) return undefined; + if (storage?.estimateBasis === 'final-fact-bytes' && value.factsBytesTotal === undefined) return undefined; + return { + ...(value.fallbackReason === undefined + ? {} + : {fallbackReason: value.fallbackReason as CodeGraphMaterializationMetrics['fallbackReason']}), + ...(value.attributionMilliseconds === undefined + ? {} + : {attributionMilliseconds: Number(value.attributionMilliseconds)}), + batchesCompleted: Number(value.batchesCompleted), + batchesTotal: Number(value.batchesTotal), + ...(value.cachedFactBytesCompleted === undefined + ? {} + : {cachedFactBytesCompleted: Number(value.cachedFactBytesCompleted)}), + ...(value.cachedFactBytesTotal === undefined ? {} : {cachedFactBytesTotal: Number(value.cachedFactBytesTotal)}), + ...(value.factsBytesCompleted === undefined ? {} : {factsBytesCompleted: Number(value.factsBytesCompleted)}), + ...(value.factsBytesTotal === undefined ? {} : {factsBytesTotal: Number(value.factsBytesTotal)}), + ...(value.loadingMilliseconds === undefined ? {} : {loadingMilliseconds: Number(value.loadingMilliseconds)}), + ...(value.mode === undefined ? {} : {mode: value.mode as CodeGraphMaterializationMetrics['mode']}), + ...(rows ? {rows} : {}), + sourceBytesCompleted: Number(value.sourceBytesCompleted), + sourceBytesTotal: Number(value.sourceBytesTotal), + ...(stageMilliseconds ? {stageMilliseconds} : {}), + ...(storage ? {storage} : {}), + ...(value.transactionMilliseconds === undefined + ? {} + : {transactionMilliseconds: Number(value.transactionMilliseconds)}), + }; +} + +function parseMaterializationStageMilliseconds( + value: unknown, +): CodeGraphMaterializationMetrics['stageMilliseconds'] | undefined { + if (!isRecord(value)) return undefined; + const stages = [ + 'attributing', + 'committing', + 'loading-cache', + 'preparing-rows', + 'writing-analysis', + 'writing-candidates', + 'writing-edges', + 'writing-facts', + 'writing-lookups', + 'writing-receipt', + 'writing-references', + 'writing-symbols', + 'writing-terms', + ] as const satisfies readonly CodeGraphMaterializationActivity['stage'][]; + const allowed = new Set(stages); + const parsed: Partial> = {}; + for (const [stage, milliseconds] of Object.entries(value)) { + if (!allowed.has(stage) || !isNonNegativeFinite(milliseconds)) return undefined; + parsed[stage as CodeGraphMaterializationActivity['stage']] = Number(milliseconds); + } + return parsed; +} + +function parseMaterializationStorage( + value: unknown, +): NonNullable | undefined { + if ( + !isRecord(value) || + !isNonNegativeSafeInteger(value.temporaryDatabaseBytes) || + !isNonNegativeSafeInteger(value.temporaryDatabaseHighWaterBytes) || + Number(value.temporaryDatabaseBytes) > Number(value.temporaryDatabaseHighWaterBytes) + ) { + return undefined; + } + if ( + value.estimateBasis !== undefined && + !['cached-fact-bytes', 'final-fact-bytes', 'source-bytes-fallback'].includes(String(value.estimateBasis)) + ) { + return undefined; + } + for (const key of [ + 'availableBytes', + 'durableAvailableBytes', + 'durableDatabaseBytes', + 'durableDatabaseFileBytes', + 'durableDatabaseFileHighWaterBytes', + 'durableDatabaseGrowthBytes', + 'durableDatabaseGrowthHighWaterBytes', + 'durableDatabaseHighWaterBytes', + 'durableDatabaseStartBytes', + 'durableFilesystemBytes', + 'durableFilesystemHighWaterBytes', + 'durableJournalBytes', + 'durableJournalHighWaterBytes', + 'durableSharedMemoryBytes', + 'durableSharedMemoryHighWaterBytes', + 'durableWalBytes', + 'durableWalHighWaterBytes', + 'estimatedConcurrentBuildBytes', + 'estimatedDurableFilesystemRequiredBytes', + 'estimatedDurableSnapshotBytes', + 'estimatedJournalBytes', + 'estimatedRequiredBytes', + 'estimatedTemporaryFilesystemRequiredBytes', + 'estimatedTemporaryDatabaseBytes', + 'temporaryAvailableBytes', + ] as const) { + if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; + } + if (value.filesystemsShared !== undefined && typeof value.filesystemsShared !== 'boolean') return undefined; + if ( + value.materializationMode !== undefined && + !['direct-persistent', 'temporary-staged'].includes(String(value.materializationMode)) + ) { + return undefined; + } + for (const [current, highWater] of [ + ['durableDatabaseFileBytes', 'durableDatabaseFileHighWaterBytes'], + ['durableDatabaseGrowthBytes', 'durableDatabaseGrowthHighWaterBytes'], + ['durableFilesystemBytes', 'durableFilesystemHighWaterBytes'], + ['durableJournalBytes', 'durableJournalHighWaterBytes'], + ['durableSharedMemoryBytes', 'durableSharedMemoryHighWaterBytes'], + ['durableWalBytes', 'durableWalHighWaterBytes'], + ] as const) { + if ( + value[current] !== undefined && + value[highWater] !== undefined && + Number(value[current]) > Number(value[highWater]) + ) { + return undefined; + } + } + if ( + value.durableDatabaseBytes !== undefined && + value.durableDatabaseHighWaterBytes !== undefined && + Number(value.durableDatabaseBytes) > Number(value.durableDatabaseHighWaterBytes) + ) { + return undefined; + } + if ( + value.estimatedRequiredBytes !== undefined && + value.estimatedConcurrentBuildBytes !== undefined && + Number(value.estimatedRequiredBytes) < Number(value.estimatedConcurrentBuildBytes) + ) { + return undefined; + } + return { + ...(value.availableBytes === undefined ? {} : {availableBytes: Number(value.availableBytes)}), + ...(value.durableAvailableBytes === undefined ? {} : {durableAvailableBytes: Number(value.durableAvailableBytes)}), + ...(value.durableDatabaseBytes === undefined ? {} : {durableDatabaseBytes: Number(value.durableDatabaseBytes)}), + ...(value.durableDatabaseFileBytes === undefined + ? {} + : {durableDatabaseFileBytes: Number(value.durableDatabaseFileBytes)}), + ...(value.durableDatabaseFileHighWaterBytes === undefined + ? {} + : {durableDatabaseFileHighWaterBytes: Number(value.durableDatabaseFileHighWaterBytes)}), + ...(value.durableDatabaseGrowthBytes === undefined + ? {} + : {durableDatabaseGrowthBytes: Number(value.durableDatabaseGrowthBytes)}), + ...(value.durableDatabaseGrowthHighWaterBytes === undefined + ? {} + : {durableDatabaseGrowthHighWaterBytes: Number(value.durableDatabaseGrowthHighWaterBytes)}), + ...(value.durableDatabaseHighWaterBytes === undefined + ? {} + : {durableDatabaseHighWaterBytes: Number(value.durableDatabaseHighWaterBytes)}), + ...(value.durableDatabaseStartBytes === undefined + ? {} + : {durableDatabaseStartBytes: Number(value.durableDatabaseStartBytes)}), + ...(value.durableFilesystemBytes === undefined + ? {} + : {durableFilesystemBytes: Number(value.durableFilesystemBytes)}), + ...(value.durableFilesystemHighWaterBytes === undefined + ? {} + : {durableFilesystemHighWaterBytes: Number(value.durableFilesystemHighWaterBytes)}), + ...(value.durableJournalBytes === undefined ? {} : {durableJournalBytes: Number(value.durableJournalBytes)}), + ...(value.durableJournalHighWaterBytes === undefined + ? {} + : {durableJournalHighWaterBytes: Number(value.durableJournalHighWaterBytes)}), + ...(value.durableSharedMemoryBytes === undefined + ? {} + : {durableSharedMemoryBytes: Number(value.durableSharedMemoryBytes)}), + ...(value.durableSharedMemoryHighWaterBytes === undefined + ? {} + : {durableSharedMemoryHighWaterBytes: Number(value.durableSharedMemoryHighWaterBytes)}), + ...(value.durableWalBytes === undefined ? {} : {durableWalBytes: Number(value.durableWalBytes)}), + ...(value.durableWalHighWaterBytes === undefined + ? {} + : {durableWalHighWaterBytes: Number(value.durableWalHighWaterBytes)}), + ...(value.estimateBasis === undefined + ? {} + : { + estimateBasis: value.estimateBasis as 'cached-fact-bytes' | 'final-fact-bytes' | 'source-bytes-fallback', + }), + ...(value.estimatedConcurrentBuildBytes === undefined + ? {} + : {estimatedConcurrentBuildBytes: Number(value.estimatedConcurrentBuildBytes)}), + ...(value.estimatedDurableFilesystemRequiredBytes === undefined + ? {} + : {estimatedDurableFilesystemRequiredBytes: Number(value.estimatedDurableFilesystemRequiredBytes)}), + ...(value.estimatedDurableSnapshotBytes === undefined + ? {} + : {estimatedDurableSnapshotBytes: Number(value.estimatedDurableSnapshotBytes)}), + ...(value.estimatedJournalBytes === undefined ? {} : {estimatedJournalBytes: Number(value.estimatedJournalBytes)}), + ...(value.estimatedRequiredBytes === undefined + ? {} + : {estimatedRequiredBytes: Number(value.estimatedRequiredBytes)}), + ...(value.estimatedTemporaryFilesystemRequiredBytes === undefined + ? {} + : {estimatedTemporaryFilesystemRequiredBytes: Number(value.estimatedTemporaryFilesystemRequiredBytes)}), + ...(value.estimatedTemporaryDatabaseBytes === undefined + ? {} + : {estimatedTemporaryDatabaseBytes: Number(value.estimatedTemporaryDatabaseBytes)}), + ...(value.filesystemsShared === undefined ? {} : {filesystemsShared: value.filesystemsShared}), + ...(value.materializationMode === undefined + ? {} + : {materializationMode: value.materializationMode as 'direct-persistent' | 'temporary-staged'}), + ...(value.temporaryAvailableBytes === undefined + ? {} + : {temporaryAvailableBytes: Number(value.temporaryAvailableBytes)}), + temporaryDatabaseBytes: Number(value.temporaryDatabaseBytes), + temporaryDatabaseHighWaterBytes: Number(value.temporaryDatabaseHighWaterBytes), + }; +} + +function parseMaterializationRows(value: unknown): CodeGraphMaterializationRows | undefined { + if (!isRecord(value)) return undefined; + const keys = [ + 'deduplicatedEdges', + 'deduplicatedReferences', + 'edges', + 'lookupKeys', + 'referenceCandidates', + 'references', + 'reexports', + 'symbols', + 'terms', + ] as const; + for (const key of keys) { + if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; + } + return Object.fromEntries(keys.flatMap(key => (value[key] === undefined ? [] : [[key, Number(value[key])]]))); +} + +function isBatchProgress(completed: unknown, total: unknown): boolean { + return isNonNegativeSafeInteger(completed) && isNonNegativeSafeInteger(total) && Number(completed) <= Number(total); +} + +function isNonNegativeSafeInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) >= 0; +} + +function parseActivity(value: unknown): CodeGraphBuildActivity | undefined { + if ( + !isRecord(value) || + !Number.isSafeInteger(value.batchCompleted) || + !Number.isSafeInteger(value.batchTotal) || + Number(value.batchCompleted) < 0 || + Number(value.batchTotal) < 0 || + Number(value.batchCompleted) > Number(value.batchTotal) || + !Number.isSafeInteger(value.bytes) || + Number(value.bytes) < 0 || + !isText(value.language, 64) || + !['extracting', 'persisting', 'reading'].includes(String(value.stage)) || + (value.classifier !== undefined && !isText(value.classifier, 64)) || + (value.degraded !== undefined && typeof value.degraded !== 'boolean') || + (value.role !== undefined && !isText(value.role, 64)) || + (value.sizeBucket !== undefined && !isCodeGraphSourceSizeBucket(value.sizeBucket)) + ) { + return undefined; + } + for (const key of ['factsBytes', 'relations', 'symbols'] as const) { + if (value[key] !== undefined && (!Number.isSafeInteger(value[key]) || Number(value[key]) < 0)) return undefined; + } + for (const key of ['parseMilliseconds', 'persistMilliseconds'] as const) { + if (value[key] !== undefined && !isNonNegativeFinite(value[key])) return undefined; + } + return { + batchCompleted: Number(value.batchCompleted), + batchTotal: Number(value.batchTotal), + bytes: Number(value.bytes), + ...(value.classifier === undefined ? {} : {classifier: value.classifier}), + ...(typeof value.degraded === 'boolean' ? {degraded: value.degraded} : {}), + ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), + language: value.language, + ...(value.parseMilliseconds === undefined ? {} : {parseMilliseconds: Number(value.parseMilliseconds)}), + ...(value.persistMilliseconds === undefined ? {} : {persistMilliseconds: Number(value.persistMilliseconds)}), + ...(value.relations === undefined ? {} : {relations: Number(value.relations)}), + ...(value.role === undefined ? {} : {role: value.role}), + ...(value.sizeBucket === undefined ? {} : {sizeBucket: value.sizeBucket}), + stage: value.stage as CodeGraphBuildActivity['stage'], + ...(value.symbols === undefined ? {} : {symbols: Number(value.symbols)}), + }; +} + +function parseExtraction(value: unknown): CodeGraphBuildExtraction | undefined { + if ( + !isRecord(value) || + !isNonNegativeSafeInteger(value.completedFiles) || + !isNonNegativeSafeInteger(value.slowFiles) || + Number(value.slowFiles) > Number(value.completedFiles) || + !Array.isArray(value.topSlowFiles) || + value.topSlowFiles.length > CODE_GRAPH_TOP_SLOW_FILE_LIMIT + ) { + return undefined; + } + const topSlowFiles = value.topSlowFiles.map(parseSlowFileTelemetry); + if (topSlowFiles.some(sample => sample === undefined)) return undefined; + const metrics = value.metrics === undefined ? undefined : parseScanningMetrics(value.metrics); + if (value.metrics !== undefined && metrics === undefined) return undefined; + const samples = topSlowFiles as CodeGraphSlowFileTelemetry[]; + if ( + samples.some( + (sample, index) => + index > 0 && + (sample.durationMilliseconds > samples[index - 1]!.durationMilliseconds || + (sample.durationMilliseconds === samples[index - 1]!.durationMilliseconds && + sample.pathHash.localeCompare(samples[index - 1]!.pathHash) < 0)), + ) + ) { + return undefined; + } + return { + completedFiles: Number(value.completedFiles), + ...(metrics === undefined ? {} : {metrics}), + slowFiles: Number(value.slowFiles), + topSlowFiles: samples, + }; +} + +function parseScanningMetrics(value: unknown): CodeGraphScanningMetrics | undefined { + if (!isRecord(value)) return undefined; + for (const key of [ + 'factsBytesCompleted', + 'sourceBytesCompleted', + 'sourceBytesTotal', + 'workUnitsCompleted', + 'workUnitsTotal', + ] as const) { + if (!isNonNegativeSafeInteger(value[key])) return undefined; + } + if ( + Number(value.sourceBytesCompleted) > Number(value.sourceBytesTotal) || + Number(value.workUnitsCompleted) > Number(value.workUnitsTotal) + ) { + return undefined; + } + return { + factsBytesCompleted: Number(value.factsBytesCompleted), + sourceBytesCompleted: Number(value.sourceBytesCompleted), + sourceBytesTotal: Number(value.sourceBytesTotal), + workUnitsCompleted: Number(value.workUnitsCompleted), + workUnitsTotal: Number(value.workUnitsTotal), + }; +} + +function parseSlowFileTelemetry(value: unknown): CodeGraphSlowFileTelemetry | undefined { + if ( + !isRecord(value) || + !isText(value.classifier, 64) || + !isNonNegativeFinite(value.durationMilliseconds) || + !isText(value.extension, 16) || + !isText(value.language, 64) || + typeof value.pathHash !== 'string' || + !/^[a-f0-9]{64}$/.test(value.pathHash) || + !isText(value.role, 64) || + !isCodeGraphSourceSizeBucket(value.sizeBucket) || + !isNonNegativeSafeInteger(value.sourceBytes) || + (value.degraded !== undefined && typeof value.degraded !== 'boolean') + ) { + return undefined; + } + for (const key of ['factsBytes', 'relations', 'symbols'] as const) { + if (value[key] !== undefined && !isNonNegativeSafeInteger(value[key])) return undefined; + } + return { + classifier: value.classifier, + ...(value.degraded === undefined ? {} : {degraded: value.degraded}), + durationMilliseconds: Number(value.durationMilliseconds), + extension: value.extension, + ...(value.factsBytes === undefined ? {} : {factsBytes: Number(value.factsBytes)}), + language: value.language, + pathHash: value.pathHash, + ...(value.relations === undefined ? {} : {relations: Number(value.relations)}), + role: value.role, + sizeBucket: value.sizeBucket, + sourceBytes: Number(value.sourceBytes), + ...(value.symbols === undefined ? {} : {symbols: Number(value.symbols)}), + }; +} + +function parseTimings(value: unknown): CodeGraphBuildTimings | undefined { + return isRecord(value) && + isNonNegativeFinite(value.extractionMilliseconds) && + isNonNegativeFinite(value.persistenceMilliseconds) && + isNonNegativeFinite(value.readingMilliseconds) + ? { + extractionMilliseconds: Number(value.extractionMilliseconds), + persistenceMilliseconds: Number(value.persistenceMilliseconds), + readingMilliseconds: Number(value.readingMilliseconds), + } + : undefined; +} + +function isNonNegativeFinite(value: unknown): boolean { + return typeof value === 'number' && Number.isFinite(value) && value >= 0; +} + +function parseRequest(value: unknown): CodeGraphBuildStatus['request'] | undefined { + return isRecord(value) && isHash(value.key) ? {key: value.key} : undefined; +} + +function parseCounters(value: unknown): CodeGraphBuildCounters | undefined { + if (!isRecord(value)) return undefined; + const keys = [ + 'accepted', + 'completed', + 'edges', + 'embedded', + 'excluded', + 'pagesCompleted', + 'reused', + 'resolved', + 'rowsDeleted', + 'skipped', + 'symbols', + 'total', + ] as const; + for (const key of keys) { + const counter = value[key]; + if (counter !== undefined && (!Number.isSafeInteger(counter) || Number(counter) < 0)) return undefined; + } + if (value.unit !== undefined && !['files', 'references', 'snapshots', 'symbols'].includes(String(value.unit))) + return undefined; + return Object.fromEntries( + [...keys, 'unit' as const].flatMap(key => (value[key] === undefined ? [] : [[key, value[key]]])), + ) as CodeGraphBuildCounters; +} + +function parseError(value: unknown): CodeGraphBuildStatus['error'] | undefined { + return isRecord(value) && isText(value.summary, 300) ? {summary: value.summary} : undefined; +} + +function parseEta(value: unknown): CodeGraphBuildStatus['eta'] | undefined { + return isRecord(value) && + value.scope === 'phase' && + ['high', 'low', 'medium'].includes(String(value.confidence)) && + (value.basis === undefined || + ['cached-fact-bytes', 'extraction-work', 'files', 'final-fact-bytes', 'source-bytes'].includes( + String(value.basis), + )) && + Number.isSafeInteger(value.remainingMilliseconds) && + Number(value.remainingMilliseconds) >= 0 + ? { + ...(value.basis === undefined + ? {} + : { + basis: value.basis as + 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes', + }), + confidence: value.confidence as 'high' | 'low' | 'medium', + remainingMilliseconds: Number(value.remainingMilliseconds), + scope: 'phase', + } + : undefined; +} + +function parseResult(value: unknown): CodeGraphBuildStatus['result'] | undefined { + if (!isRecord(value) || typeof value.dirty !== 'boolean' || !isText(value.snapshotId, 128)) return undefined; + for (const key of ['edges', 'files', 'symbols'] as const) { + if (!Number.isSafeInteger(value[key]) || Number(value[key]) < 0) return undefined; + } + return { + dirty: value.dirty, + edges: Number(value.edges), + files: Number(value.files), + snapshotId: value.snapshotId, + symbols: Number(value.symbols), + }; +} diff --git a/src/code_graph/build_status_validation.ts b/src/code_graph/build_status_validation.ts new file mode 100644 index 00000000..2c12ddec --- /dev/null +++ b/src/code_graph/build_status_validation.ts @@ -0,0 +1,21 @@ +export const CODE_GRAPH_BUILD_STATUS_SCHEMA_VERSION = 1 as const; + +export const CODE_GRAPH_BUILD_HASH_ID = /^[0-9a-f]{64}$/; +export const CODE_GRAPH_BUILD_ID = /^[0-9a-f-]{16,64}$/; +export const CODE_GRAPH_BUILD_COMMIT_ID = /^[0-9a-f]{7,64}$/; + +export function isBuildStatusRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isBuildStatusHash(value: unknown): value is string { + return typeof value === 'string' && CODE_GRAPH_BUILD_HASH_ID.test(value); +} + +export function isBuildStatusText(value: unknown, maximum: number): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= maximum && !/[\p{Cc}]/u.test(value); +} + +export function isBuildStatusTimestamp(value: unknown): value is string { + return isBuildStatusText(value, 64) && Number.isFinite(Date.parse(value)); +} diff --git a/src/code_graph/commands.ts b/src/code_graph/commands.ts index c50268b3..b74841e6 100644 --- a/src/code_graph/commands.ts +++ b/src/code_graph/commands.ts @@ -86,6 +86,10 @@ export interface CodeGraphCliReadPlan { readonly unavailable: boolean; } +class CodeGraphCommandError extends Error { + readonly _tag = 'CodeGraphCommandError' as const; +} + export const CODE_GRAPH_CLI_READ_TIMEOUT_MILLISECONDS = 25_000; const CODE_GRAPH_CLI_READ_RETRY_MILLISECONDS = 1_000; @@ -201,10 +205,10 @@ export const runCodeGraphRepair = Effect.fn('codeGraph.command.repair')(function }, ) { if (options.all && (options.checkoutId !== undefined || options.cwd !== undefined)) { - return yield* Effect.fail(new Error('Use --all by itself, without --checkout-id or --cwd.')); + return yield* Effect.fail(new CodeGraphCommandError('Use --all by itself, without --checkout-id or --cwd.')); } if (options.checkoutId !== undefined && options.cwd !== undefined) { - return yield* Effect.fail(new Error('Use either --checkout-id or --cwd, not both.')); + return yield* Effect.fail(new CodeGraphCommandError('Use either --checkout-id or --cwd, not both.')); } const targetCheckoutId = options.all ? undefined @@ -755,7 +759,9 @@ export const runCodeGraphIndex = Effect.fn('codeGraph.command.index')(function* const cwd = yield* commandCwd(options.cwd); const identity = yield* resolveRepositoryIdentity(cwd); if (options.expectedIdentity && !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity)) { - return yield* Effect.fail(new Error('Repository identity does not match the requested graph target.')); + return yield* Effect.fail( + new CodeGraphCommandError('Repository identity does not match the requested graph target.'), + ); } const ensureVectors = options.noVectors === true ? false : undefined; if (options.json) { @@ -797,7 +803,7 @@ export const runCodeGraphIndex = Effect.fn('codeGraph.command.index')(function* .pipe(Effect.catch(() => Effect.void)), ), ), - progress => progress.stop().pipe(Effect.catch(() => Effect.void)), + progress => progress.stop.pipe(Effect.catch(() => Effect.void)), ).pipe( Effect.flatMap(summary => Console.log( @@ -819,7 +825,9 @@ export const runCodeGraphWorksetPrepare = Effect.fn('codeGraph.command.worksetPr ); if (result.state === 'failed') { return yield* Effect.fail( - new Error('Workset preparation was incomplete; the previous published catalog generation was preserved.'), + new CodeGraphCommandError( + 'Workset preparation was incomplete; the previous published catalog generation was preserved.', + ), ); } }); @@ -890,7 +898,9 @@ export const runCodeGraphAnalysis = Effect.fn('codeGraph.command.analysis')(func const communityId = options.communityId?.trim(); if (options.view === 'community' && !communityId?.match(/^cgc_[a-f0-9]{32}$/)) { return yield* Effect.fail( - new Error('Community drill-down requires --community-id with a stable cgc_ identifier from graph communities.'), + new CodeGraphCommandError( + 'Community drill-down requires --community-id with a stable cgc_ identifier from graph communities.', + ), ); } const status = yield* ensureAnalysisSnapshot(config, cwd, options.json === true); @@ -934,7 +944,8 @@ export const runCodeGraphReport = Effect.fn('codeGraph.command.report')(function const path = yield* Path.Path; const cwd = yield* commandCwd(options.cwd); const output = path.resolve(options.output); - if (yield* fs.exists(output)) return yield* Effect.fail(new Error(`Report output already exists: ${output}`)); + if (yield* fs.exists(output)) + return yield* Effect.fail(new CodeGraphCommandError(`Report output already exists: ${output}`)); const status = yield* ensureAnalysisSnapshot(config, cwd, false); const analysis = yield* CodeGraphAnalysis; const result = yield* analysis.analyze({ @@ -1000,7 +1011,7 @@ export const runCodeGraphInspect = Effect.fn('codeGraph.command.inspect')(functi } if (options.operation === 'impact') { const query = options.query?.trim(); - if (!query) return yield* Effect.fail(new Error('A workset impact trace requires --query.')); + if (!query) return yield* Effect.fail(new CodeGraphCommandError('A workset impact trace requires --query.')); const result = yield* traceCodeGraphWorksetImpact(config, { maxDepth: options.depth, maxEdges: options.edgeLimit, @@ -1011,7 +1022,7 @@ export const runCodeGraphInspect = Effect.fn('codeGraph.command.inspect')(functi return; } if (options.operation !== 'query') { - return yield* Effect.fail(new Error('--workset is valid for graph query, path, and impact.')); + return yield* Effect.fail(new CodeGraphCommandError('--workset is valid for graph query, path, and impact.')); } const cursor = options.cursor?.trim(); const projected = cursor @@ -1021,7 +1032,8 @@ export const runCodeGraphInspect = Effect.fn('codeGraph.command.inspect')(functi }) : yield* Effect.gen(function* () { const query = options.query?.trim(); - if (!query) return yield* Effect.fail(new Error('A workset graph query requires --query or --cursor.')); + if (!query) + return yield* Effect.fail(new CodeGraphCommandError('A workset graph query requires --query or --cursor.')); return yield* queryCodeGraphWorksetV2(config, { depth: options.depth, edgeLimit: options.edgeLimit, @@ -1038,10 +1050,10 @@ export const runCodeGraphInspect = Effect.fn('codeGraph.command.inspect')(functi return; } if (options.cursor?.trim() || options.budgetTokens !== undefined) { - return yield* Effect.fail(new Error('--cursor and --budget-tokens require --workset.')); + return yield* Effect.fail(new CodeGraphCommandError('--cursor and --budget-tokens require --workset.')); } if (options.operation === 'query' && !options.query?.trim()) { - return yield* Effect.fail(new Error('A graph query requires --query.')); + return yield* Effect.fail(new CodeGraphCommandError('A graph query requires --query.')); } const qualifiedTarget = options.nodeId?.startsWith('cgr_') ? yield* resolveCodeGraphQualifiedRefTarget(config, options.nodeId, options.cwd) @@ -1082,7 +1094,7 @@ export const runCodeGraphInspect = Effect.fn('codeGraph.command.inspect')(functi ? Effect.acquireUseRelease( startProgress('Scanning repository source from Git.'), progress => inspect(state => progress.update(progressMessage(state)).pipe(Effect.catch(() => Effect.void))), - progress => progress.stop().pipe(Effect.catch(() => Effect.void)), + progress => progress.stop.pipe(Effect.catch(() => Effect.void)), ) : inspect(); const readTimeoutMilliseconds = options.readTimeoutMilliseconds ?? CODE_GRAPH_CLI_READ_TIMEOUT_MILLISECONDS; @@ -1181,7 +1193,9 @@ export const runCodeGraphImpact = Effect.fn('codeGraph.command.impact')(function ) { if (options.workset?.trim()) { if (!options.query?.trim()) { - return yield* Effect.fail(new Error('A workset impact trace requires --query with a qualified endpoint.')); + return yield* Effect.fail( + new CodeGraphCommandError('A workset impact trace requires --query with a qualified endpoint.'), + ); } yield* runCodeGraphInspect(config, {...options, operation: 'impact'}); return; @@ -1215,14 +1229,18 @@ export const runCodeGraphPurge = Effect.fn('codeGraph.command.purge')(function* ) { const path = yield* Path.Path; if (options.all && (options.checkoutId !== undefined || options.obsolete || options.snapshotId !== undefined)) { - return yield* Effect.fail(new Error('Use --all by itself, without --checkout-id, --obsolete, or --snapshot-id.')); + return yield* Effect.fail( + new CodeGraphCommandError('Use --all by itself, without --checkout-id, --obsolete, or --snapshot-id.'), + ); } if (options.checkoutId !== undefined && options.cwd !== undefined) { - return yield* Effect.fail(new Error('Use either --checkout-id or --cwd, not both.')); + return yield* Effect.fail(new CodeGraphCommandError('Use either --checkout-id or --cwd, not both.')); } if (options.snapshotId !== undefined) { if (options.obsolete || options.all || options.dryRun) { - return yield* Effect.fail(new Error('Use --snapshot-id without --all, --obsolete, or --dry-run.')); + return yield* Effect.fail( + new CodeGraphCommandError('Use --snapshot-id without --all, --obsolete, or --dry-run.'), + ); } let checkoutId = options.checkoutId; if (checkoutId === undefined) { @@ -1242,7 +1260,7 @@ export const runCodeGraphPurge = Effect.fn('codeGraph.command.purge')(function* return; } if (options.apply || options.approval !== undefined || options.json) { - return yield* Effect.fail(new Error('Use --apply, --approval, or --json only with --snapshot-id.')); + return yield* Effect.fail(new CodeGraphCommandError('Use --apply, --approval, or --json only with --snapshot-id.')); } if (options.obsolete) { let checkoutId = options.checkoutId; @@ -1345,7 +1363,9 @@ export const runCodeGraphCompact = Effect.fn('codeGraph.command.compact')(functi const cwd = yield* commandCwd(options.cwd); const identity = yield* resolveRepositoryIdentity(cwd); if (options.expectedIdentity && !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity)) { - return yield* Effect.fail(new Error('Repository identity does not match the requested graph target.')); + return yield* Effect.fail( + new CodeGraphCommandError('Repository identity does not match the requested graph target.'), + ); } const summary = yield* compactCodeGraphStorage(config.agentContextHome, identity.checkoutId, { dryRun: options.dryRun === true, @@ -1403,11 +1423,14 @@ export const runCodeGraphExport = Effect.fn('codeGraph.command.export')(function const snapshot = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); if (!snapshot) { return yield* Effect.fail( - new Error('No ready native code graph snapshot exists. Run `threadnote graph index` before exporting.'), + new CodeGraphCommandError( + 'No ready native code graph snapshot exists. Run `threadnote graph index` before exporting.', + ), ); } const output = path.resolve(options.output); - if (yield* fs.exists(output)) return yield* Effect.fail(new Error(`Export output already exists: ${output}`)); + if (yield* fs.exists(output)) + return yield* Effect.fail(new CodeGraphCommandError(`Export output already exists: ${output}`)); yield* options.interlock?.afterOutputCheck?.() ?? Effect.void; const edgeLimit = yield* parseCodeGraphExportLimit(options.edgeLimit, '--edge-limit'); const nodeLimit = yield* parseCodeGraphExportLimit(options.nodeLimit, '--node-limit'); @@ -1438,7 +1461,7 @@ export const runCodeGraphExport = Effect.fn('codeGraph.command.export')(function const linked = yield* fs.link(temporary, output).pipe(Effect.result); if (linked._tag === 'Failure') { if (yield* fs.exists(output)) { - return yield* Effect.fail(new Error(`Export output already exists: ${output}`)); + return yield* Effect.fail(new CodeGraphCommandError(`Export output already exists: ${output}`)); } return yield* linked.failure; } @@ -1488,7 +1511,7 @@ function parseCodeGraphExportLimit( if (value === undefined || value === 'all') return Effect.succeed(value); const parsed = typeof value === 'number' ? value : Number(value.trim()); if (!Number.isSafeInteger(parsed) || parsed < 0 || (typeof value === 'string' && !/^\d+$/.test(value.trim()))) { - return Effect.fail(new Error(`${flag} must be "all" or a non-negative safe integer.`)); + return Effect.fail(new CodeGraphCommandError(`${flag} must be "all" or a non-negative safe integer.`)); } return Effect.succeed(parsed); } @@ -1500,11 +1523,13 @@ function verifyOwnedExportTemporary( ) { return Effect.gen(function* () { if (Option.isSome(yield* fs.readLink(temporary).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Export temporary path was replaced by a symbolic link.')); + return yield* Effect.fail(new CodeGraphCommandError('Export temporary path was replaced by a symbolic link.')); } const current = exportTemporaryIdentity(yield* fs.stat(temporary)); if (Option.isNone(current) || !sameExportFile(expected, current.value)) { - return yield* Effect.fail(new Error('Export temporary path no longer identifies the private output file.')); + return yield* Effect.fail( + new CodeGraphCommandError('Export temporary path no longer identifies the private output file.'), + ); } }); } @@ -1517,7 +1542,7 @@ function verifyPublishedExportOutput( return Effect.gen(function* () { if (Option.isSome(yield* fs.readLink(output).pipe(Effect.option))) { yield* fs.remove(output, {force: true}); - return yield* Effect.fail(new Error('Export publication did not link the private output file.')); + return yield* Effect.fail(new CodeGraphCommandError('Export publication did not link the private output file.')); } const current = yield* fs.stat(output).pipe(Effect.option); const identity = Option.flatMap(current, exportTemporaryIdentity); @@ -1530,10 +1555,10 @@ function verifyPublishedExportOutput( } if (Option.isSome(yield* fs.readLink(output).pipe(Effect.option))) { yield* fs.remove(output, {force: true}); - return yield* Effect.fail(new Error('Export publication did not link the private output file.')); + return yield* Effect.fail(new CodeGraphCommandError('Export publication did not link the private output file.')); } if (Option.isSome(identity)) yield* removeOwnedExportTemporary(fs, output, identity.value); - return yield* Effect.fail(new Error('Export publication did not link the private output file.')); + return yield* Effect.fail(new CodeGraphCommandError('Export publication did not link the private output file.')); }); } @@ -1566,7 +1591,9 @@ function removeOpenedExportTemporary( function requireExportTemporaryIdentity(info: FileSystem.File.Info) { return Option.match(exportTemporaryIdentity(info), { onNone: () => - Effect.fail(new Error('Export temporary file has insufficient identity metadata for safe publication.')), + Effect.fail( + new CodeGraphCommandError('Export temporary file has insufficient identity metadata for safe publication.'), + ), onSome: Effect.succeed, }); } @@ -1641,13 +1668,13 @@ const ensureAnalysisSnapshot = Effect.fn('codeGraph.command.ensureAnalysisSnapsh onProgress: state => progress.update(progressMessage(state)).pipe(Effect.catch(() => Effect.void)), threadnoteHome: config.agentContextHome, }), - progress => progress.stop().pipe(Effect.catch(() => Effect.void)), + progress => progress.stop.pipe(Effect.catch(() => Effect.void)), ); } status = yield* query.status(config.agentContextHome, cwd); } if (!status.readySnapshot) { - return yield* Effect.fail(new Error('No ready native code graph snapshot exists after indexing.')); + return yield* Effect.fail(new CodeGraphCommandError('No ready native code graph snapshot exists after indexing.')); } return { databasePath: status.databasePath, diff --git a/src/code_graph/cross_repository/query_expansion.ts b/src/code_graph/cross_repository/query_expansion.ts index 1dc3e4a1..174a3b2d 100644 --- a/src/code_graph/cross_repository/query_expansion.ts +++ b/src/code_graph/cross_repository/query_expansion.ts @@ -16,6 +16,10 @@ import { type CodeGraphCrossRepositoryBridgeCursorV1, } from './store.js'; +class CodeGraphQueryExpansionError extends Error { + readonly _tag = 'CodeGraphQueryExpansionError' as const; +} + const DEFAULT_SEED_REPOSITORIES = 16; const MAXIMUM_BRIDGES_PER_SEED_DIRECTION = 64; const BRIDGE_PAGE_SIZE = 64; @@ -72,11 +76,18 @@ export const readCodeGraphWorksetQueryBridgeExpansion = Effect.fn( seed => { const member = memberByKey.get(seed.repositoryKey); if (member === undefined || member.repositoryId !== seed.repositoryId || member.snapshotId !== seed.snapshotId) { - return Effect.fail(new Error('A routed bridge seed does not match the published generation.')); + return Effect.fail( + new CodeGraphQueryExpansionError('A routed bridge seed does not match the published generation.'), + ); } return Effect.forEach( ['outgoing', 'incoming'] as const, - direction => readSeedDirection(threadnoteHome, published, member, direction, bridgeSet), + direction => + readSeedDirection(threadnoteHome, published, member, direction, bridgeSet).pipe( + Effect.mapError( + cause => new CodeGraphQueryExpansionError('Could not read the published bridge expansion.', {cause}), + ), + ), {concurrency: 2}, ); }, @@ -92,7 +103,7 @@ export const readCodeGraphWorksetQueryBridgeExpansion = Effect.fn( } const bridges = [...byId.values()].sort(compareBridge); if (bridges.length > MAXIMUM_EXPANSION_BRIDGES) { - throw new Error('Workset query bridge expansion exceeded its deterministic bound.'); + throw new CodeGraphQueryExpansionError('Workset query bridge expansion exceeded its deterministic bound.'); } return { bridgeSet, @@ -251,7 +262,7 @@ export function materializeCodeGraphWorksetBridgeEndpointCards( maximumCards = 4, ): readonly CodeGraphEvidenceCardV1[] { if (!Number.isSafeInteger(maximumCards) || maximumCards < 0 || maximumCards > 32) { - throw new Error('Workset bridge endpoint card limit is invalid.'); + throw new CodeGraphQueryExpansionError('Workset bridge endpoint card limit is invalid.'); } if (maximumCards === 0) return []; const members = new Map(published.members.map(member => [member.repositoryKey, member] as const)); @@ -265,7 +276,7 @@ export function materializeCodeGraphWorksetBridgeEndpointCards( member.repositoryId !== endpoint.repositoryId || member.snapshotId !== endpoint.snapshotId ) { - throw new Error('A bridge endpoint card is outside its published generation.'); + throw new CodeGraphQueryExpansionError('A bridge endpoint card is outside its published generation.'); } if (byRef.has(endpoint.reference.ref)) continue; const qualifiedName = bridge.identity.replace(/^protobuf:[^:]+:/u, ''); @@ -305,7 +316,7 @@ export function mergeCodeGraphWorksetBridgeEndpointCards( maximumCards: number, ): readonly CodeGraphEvidenceCardV1[] { if (!Number.isSafeInteger(maximumCards) || maximumCards < 1 || maximumCards > 512) { - throw new Error('Workset evidence card limit is invalid.'); + throw new CodeGraphQueryExpansionError('Workset evidence card limit is invalid.'); } const output: CodeGraphEvidenceCardV1[] = []; const refs = new Set(); @@ -356,7 +367,9 @@ function readSeedDirection( page.totalBridges !== bridgeSet.totalBridges || page.coverage.state !== 'complete' ) { - throw new Error('The published bridge set changed or became incomplete during query expansion.'); + throw new CodeGraphQueryExpansionError( + 'The published bridge set changed or became incomplete during query expansion.', + ); } bridges.push(...page.bridges); after = page.next; @@ -381,7 +394,8 @@ function registerBridgeCandidate( bridge: CodeGraphCrossRepositoryBridgeV1, ): void { const member = publishedByKey.get(repositoryKey); - if (member === undefined) throw new Error('A bridge neighbor is absent from its published generation.'); + if (member === undefined) + throw new CodeGraphQueryExpansionError('A bridge neighbor is absent from its published generation.'); const existing = additions.get(repositoryKey); if (existing === undefined) { additions.set(repositoryKey, { @@ -410,20 +424,20 @@ function validateExpansion( expansion: CodeGraphWorksetQueryBridgeExpansionV1, ): void { if (!Array.isArray(expansion.bridges) || expansion.bridges.length > MAXIMUM_EXPANSION_BRIDGES) { - throw new Error('Workset query bridge expansion exceeds its supported bound.'); + throw new CodeGraphQueryExpansionError('Workset query bridge expansion exceeds its supported bound.'); } if ( !Number.isSafeInteger(expansion.seededRepositories) || expansion.seededRepositories < 0 || expansion.seededRepositories > Math.min(DEFAULT_SEED_REPOSITORIES, router.repositories.length) ) { - throw new Error('Workset query bridge seed coverage is invalid.'); + throw new CodeGraphQueryExpansionError('Workset query bridge seed coverage is invalid.'); } if (expansion.bridges.length > 0 && expansion.bridgeSet === undefined) { - throw new Error('Workset query bridges have no generation receipt.'); + throw new CodeGraphQueryExpansionError('Workset query bridges have no generation receipt.'); } if (expansion.bridgeSet !== undefined && expansion.bridgeSet.generationId !== published.id) { - throw new Error('Workset query bridges belong to another generation.'); + throw new CodeGraphQueryExpansionError('Workset query bridges belong to another generation.'); } const members = new Map( published.members.map(member => [`${member.repositoryId}\0${member.snapshotId}`, member] as const), @@ -431,16 +445,17 @@ function validateExpansion( const seeds = new Set(router.repositories.map(repository => repository.repositoryKey)); const seen = new Set(); for (const bridge of expansion.bridges) { - if (seen.has(bridge.id)) throw new Error('Workset query bridge expansion contains a duplicate edge.'); + if (seen.has(bridge.id)) + throw new CodeGraphQueryExpansionError('Workset query bridge expansion contains a duplicate edge.'); seen.add(bridge.id); for (const endpoint of [bridge.source, bridge.target]) { const member = members.get(`${endpoint.repositoryId}\0${endpoint.snapshotId}`); if (member === undefined || member.repositoryKey !== endpoint.repositoryKey) { - throw new Error('Workset query bridge endpoint is outside the published generation.'); + throw new CodeGraphQueryExpansionError('Workset query bridge endpoint is outside the published generation.'); } } if (!seeds.has(bridge.source.repositoryKey) && !seeds.has(bridge.target.repositoryKey)) { - throw new Error('Workset query bridge is not adjacent to a routed repository.'); + throw new CodeGraphQueryExpansionError('Workset query bridge is not adjacent to a routed repository.'); } } } diff --git a/src/code_graph/cross_repository/runtime.ts b/src/code_graph/cross_repository/runtime.ts index c06da10c..02a956d6 100644 --- a/src/code_graph/cross_repository/runtime.ts +++ b/src/code_graph/cross_repository/runtime.ts @@ -37,6 +37,10 @@ import { type CodeGraphCrossRepositoryTraversalEndpointV1, } from './traversal.js'; +class CodeGraphCrossRepositoryRuntimeError extends Error { + readonly _tag = 'CodeGraphCrossRepositoryRuntimeError' as const; +} + const SNAPSHOT_LEASE_MILLISECONDS = 2 * 60_000; const LOCAL_ADJACENCY_SCAN_MAXIMUM = 5_000; const TOPOLOGY_BRIDGES_MAXIMUM_DEFAULT = 20_000; @@ -210,13 +214,17 @@ const inspectCodeGraphWorksetTopologyScoped = Effect.fn('codeGraphCrossRepositor page.totalBridges !== bridgeSet.bridgeCount || page.coverage.state !== 'complete' ) { - throw new Error('The published bridge set changed or became unavailable during topology assembly.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'The published bridge set changed or became unavailable during topology assembly.', + ); } bridges.push(...page.bridges); after = page.next; } while (after !== undefined); if (bridges.length !== bridgeSet.bridgeCount) { - throw new Error('The complete bridge topology page sequence does not match its receipt.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'The complete bridge topology page sequence does not match its receipt.', + ); } const topology = projectCodeGraphCrossRepositoryTopology({ bridgeSet: { @@ -259,12 +267,16 @@ function prepareRuntime(config: RuntimeConfig, worksetName: string) { const published = yield* readPublishedCodeGraphWorksetCatalogGeneration(config.agentContextHome, workset.name); if (published === undefined) { return yield* Effect.fail( - new Error(`No published workset catalog exists for ${workset.name}; run \`threadnote workset prepare\`.`), + new CodeGraphCrossRepositoryRuntimeError( + `No published workset catalog exists for ${workset.name}; run \`threadnote workset prepare\`.`, + ), ); } if (!codeGraphWorksetCatalogGenerationMatches(workset, manifestDigest, published)) { return yield* Effect.fail( - new Error(`The published workset catalog for ${workset.name} is stale; run \`threadnote workset prepare\`.`), + new CodeGraphCrossRepositoryRuntimeError( + `The published workset catalog for ${workset.name} is stale; run \`threadnote workset prepare\`.`, + ), ); } const projectsByKey = new Map(workset.projects.map(project => [safeLabel(project.name), project] as const)); @@ -301,11 +313,17 @@ function requireCompleteBridgeSet(config: RuntimeConfig, runtime: PreparedRuntim Effect.flatMap(bridgeSet => { if (bridgeSet === undefined) { return Effect.fail( - new Error('The published workset generation has no cross-repository bridge receipt; run workset prepare.'), + new CodeGraphCrossRepositoryRuntimeError( + 'The published workset generation has no cross-repository bridge receipt; run workset prepare.', + ), ); } if (bridgeSet.coverage.state !== 'complete') { - return Effect.fail(new Error('Cross-repository bridge coverage is incomplete; path and impact were withheld.')); + return Effect.fail( + new CodeGraphCrossRepositoryRuntimeError( + 'Cross-repository bridge coverage is incomplete; path and impact were withheld.', + ), + ); } return Effect.succeed(bridgeSet); }), @@ -418,7 +436,9 @@ function readLocalAdjacencyPage( const sourceMayHaveMore = rows.length === requested; if (selected.length >= limit || !sourceMayHaveMore || requested === LOCAL_ADJACENCY_SCAN_MAXIMUM) { if (selected.length === 0 && sourceMayHaveMore && requested === LOCAL_ADJACENCY_SCAN_MAXIMUM) { - throw new Error('Local adjacency exceeded the bounded scan before yielding a traversable edge.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'Local adjacency exceeded the bounded scan before yielding a traversable edge.', + ); } for (const {edge} of selected) { for (const candidate of [edge.source, edge.target]) { @@ -503,17 +523,24 @@ function resolveTraversalEndpoint(config: RuntimeConfig, runtime: PreparedRuntim const record = yield* resolveCodeGraphQualifiedRef(config.agentContextHome, {ref: normalized}); const member = runtime.published.members.find(candidate => candidate.repositoryId === record.repositoryId); if (member === undefined) - throw new Error('The qualified reference repository is not in this workset generation.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'The qualified reference repository is not in this workset generation.', + ); const present = yield* codeGraphWorksetCatalogProjectionContainsNode(config.agentContextHome, { nodeId: record.nodeId, projectionDigest: member.projectionDigest, }); - if (!present) throw new Error('The qualified reference is not present in the published snapshot projection.'); + if (!present) + throw new CodeGraphCrossRepositoryRuntimeError( + 'The qualified reference is not present in the published snapshot projection.', + ); return traversalEndpoint(member, {kind: 'qualified-ref', ref: normalized}); } if (COMPONENT_ID.test(normalized)) { if (runtime.published.members.length !== 1) { - throw new Error('A component selector in a multi-repository workset must use :.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'A component selector in a multi-repository workset must use :.', + ); } return traversalEndpoint(runtime.published.members[0]!, {componentId: normalized, kind: 'component'}); } @@ -521,12 +548,18 @@ function resolveTraversalEndpoint(config: RuntimeConfig, runtime: PreparedRuntim if (marker > 0) { const repositoryKey = normalized.slice(0, marker); const componentId = normalized.slice(marker + 1); - if (!COMPONENT_ID.test(componentId)) throw new Error('Workset component selector is invalid.'); + if (!COMPONENT_ID.test(componentId)) + throw new CodeGraphCrossRepositoryRuntimeError('Workset component selector is invalid.'); const member = runtime.published.members.find(candidate => candidate.repositoryKey === repositoryKey); - if (member === undefined) throw new Error('Workset component selector names an unknown generation member.'); + if (member === undefined) + throw new CodeGraphCrossRepositoryRuntimeError( + 'Workset component selector names an unknown generation member.', + ); return traversalEndpoint(member, {componentId, kind: 'component'}); } - throw new Error('Workset path/impact requires a cgr_ handle or : component selector.'); + throw new CodeGraphCrossRepositoryRuntimeError( + 'Workset path/impact requires a cgr_ handle or : component selector.', + ); }); } @@ -559,7 +592,8 @@ function repositorySnapshotKey(value: {readonly repositoryId: string; readonly s function localOffset(cursor: string | undefined): number { if (cursor === undefined) return 0; - if (!/^(?:0|[1-9]\d{0,3})$/u.test(cursor)) throw new Error('Local traversal cursor is invalid.'); + if (!/^(?:0|[1-9]\d{0,3})$/u.test(cursor)) + throw new CodeGraphCrossRepositoryRuntimeError('Local traversal cursor is invalid.'); return boundedInteger(Number(cursor), 'local traversal cursor', 0, LOCAL_ADJACENCY_SCAN_MAXIMUM); } @@ -570,7 +604,7 @@ function safeLabel(value: string): string { function boundedInteger(value: number, label: string, minimum: number, maximum: number): number { if (!Number.isSafeInteger(value) || value < minimum || value > maximum) { - throw new Error(`${label} must be an integer from ${minimum} to ${maximum}.`); + throw new CodeGraphCrossRepositoryRuntimeError(`${label} must be an integer from ${minimum} to ${maximum}.`); } return value; } diff --git a/src/code_graph/cross_repository/store.ts b/src/code_graph/cross_repository/store.ts index 1e4b5e36..e0816c71 100644 --- a/src/code_graph/cross_repository/store.ts +++ b/src/code_graph/cross_repository/store.ts @@ -1,7 +1,10 @@ -import {Clock, Effect} from 'effect'; +import {Clock, Effect, Path} from 'effect'; import * as SqlClient from 'effect/unstable/sql/SqlClient'; import {sha256HexSync} from '../../crypto/sha256.js'; +import {SystemInfo} from '../../effect/system.js'; import {compareCodeUnits} from '../ordering.js'; +import {codeGraphWorksetCatalogLayout} from '../workset_catalog/layout.js'; +import {changes} from '../workset_catalog/store_support.js'; import {CODE_GRAPH_WORKSET_CATALOG_LIMITS, CodeGraphWorksetCatalogError} from '../workset_catalog/types.js'; import {withCodeGraphWorksetCatalogReader, withCodeGraphWorksetCatalogWriter} from '../workset_catalog/store.js'; import { @@ -23,6 +26,10 @@ const MAX_REPOSITORY_KEY_BYTES = 4_096; const MAX_SNAPSHOT_ID_BYTES = 256; const MAX_IDENTITY_BYTES = 8_192; const MAX_EVIDENCE_PATH_BYTES = 4_096; +const BRIDGE_SET_DISK_SAFETY_BYTES = 512 * 1_024 * 1_024; +const BRIDGE_ROW_STORAGE_OVERHEAD_BYTES = 1_024; +const BRIDGE_SET_WRITE_AMPLIFICATION = 2; +const BRIDGE_SET_DIGEST_DOMAIN = 'threadnote-cross-repository-bridge-set-v1'; export interface CodeGraphCrossRepositoryEndpointKeyV1 { readonly reference: CodeGraphBridgeEndpointReferenceV1; @@ -105,6 +112,7 @@ interface GenerationMemberRow { } interface BridgeSetRow { + readonly bridge_bytes: unknown; readonly bridge_count: unknown; readonly bridge_set_digest: unknown; readonly coverage_state: unknown; @@ -155,6 +163,17 @@ interface PreparedBridge { readonly json: string; } +interface PreparedBridgeSet { + readonly bridges: readonly PreparedBridge[]; + readonly digest: string; + readonly totalBytes: number; +} + +interface StoredBridgeFootprint { + readonly bridgeCount: number; + readonly totalBytes: number; +} + /** * Atomically replace the complete bridge set for one deterministic generation. * A staged generation stays invisible. A ready generation must still be the @@ -168,10 +187,15 @@ export const replaceCodeGraphWorksetCatalogBridgeSet = Effect.fn('codeGraphCross readonly coverage?: Omit & { readonly repositoryCount?: number; }; + /** @internal Deterministic capacity probe used by focused storage tests. */ + readonly diskCapacityAvailableBytes?: (target: string) => Effect.Effect; readonly generationId: string; }, ) { const prepared = yield* validateInput(() => prepareBridgeSet(input)); + const path = yield* Path.Path; + const system = yield* SystemInfo; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); return yield* withCodeGraphWorksetCatalogWriter(threadnoteHome, sql => Effect.gen(function* () { const generation = yield* loadWritableGeneration(sql, input.generationId); @@ -180,21 +204,72 @@ export const replaceCodeGraphWorksetCatalogBridgeSet = Effect.fn('codeGraphCross const coverage = yield* validateInput(() => prepareCoverage(input.coverage, members.length, prepared.bridges.length), ); + const stored = yield* loadStoredBridgeFootprint(sql, input.generationId); + const requiredFreeBytes = yield* validateInput(() => + codeGraphCrossRepositoryBridgeReplacementRequiredFreeBytes({ + existingBridgeBytes: stored.totalBytes, + existingBridgeCount: stored.bridgeCount, + replacementBridgeBytes: prepared.totalBytes, + replacementBridgeCount: prepared.bridges.length, + }), + ); + yield* verifyBridgeReplacementDiskCapacity( + input.diskCapacityAvailableBytes ?? (target => system.availableDiskBytes(target)), + layout.root, + requiredFreeBytes, + ); const replacedAt = yield* currentIsoInstant; yield* sql.withTransaction( Effect.gen(function* () { + const capacities = yield* sql.unsafe<{ + readonly bridge_logical_bytes: unknown; + readonly projection_logical_bytes: unknown; + }>( + `SELECT bridge_logical_bytes, projection_logical_bytes + FROM catalog_capacity WHERE singleton = 1 LIMIT 1`, + ); + if (capacities.length !== 1) { + return yield* Effect.fail(corrupt('Catalog capacity receipt is missing.')); + } + const bridgeLogicalBytes = requiredInteger( + capacities[0]!.bridge_logical_bytes, + 'catalog bridge logical bytes', + ); + const projectionLogicalBytes = requiredInteger( + capacities[0]!.projection_logical_bytes, + 'catalog projection logical bytes', + ); + const nextBridgeLogicalBytes = bridgeLogicalBytes - stored.totalBytes + prepared.totalBytes; + if ( + nextBridgeLogicalBytes < 0 || + nextBridgeLogicalBytes + projectionLogicalBytes > + CODE_GRAPH_WORKSET_CATALOG_LIMITS.catalogPhysicalBytesMaximum + ) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('capacity', 'The home-global workset catalog is full.'), + ); + } + yield* sql.unsafe( + `UPDATE catalog_capacity SET bridge_logical_bytes = ? + WHERE singleton = 1 AND bridge_logical_bytes = ?`, + [nextBridgeLogicalBytes, bridgeLogicalBytes], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Catalog bridge capacity receipt changed unexpectedly.')); + } yield* sql.unsafe('DELETE FROM cross_repository_bridge_sets WHERE generation_id = ?', [input.generationId]); yield* sql.unsafe( `INSERT INTO cross_repository_bridge_sets ( - generation_id, resolver_version, bridge_count, bridge_set_digest, + generation_id, resolver_version, bridge_count, bridge_bytes, bridge_set_digest, coverage_state, repository_count, repositories_read, failed_repository_count, rejection_count, diagnostic_codes_json, replaced_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ input.generationId, CODE_GRAPH_CROSS_REPOSITORY_RESOLVER_VERSION, prepared.bridges.length, + prepared.totalBytes, prepared.digest, coverage.state, coverage.repositoryCount, @@ -465,12 +540,13 @@ export const readCodeGraphWorksetCatalogBridgeGenerationPage = Effect.fn( function prepareBridgeSet(input: { readonly bridges: readonly CodeGraphCrossRepositoryBridgeV1[]; readonly generationId: string; -}) { +}): PreparedBridgeSet { if (!GENERATION_ID.test(input.generationId)) throw invalid('Bridge generation identity is invalid.'); if (!Array.isArray(input.bridges) || input.bridges.length > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration) { throw invalid('Bridge set exceeds the supported generation bound.'); } const byId = new Map(); + let totalBytes = 0; for (const value of input.bridges) { const bridge = parseCanonicalBridge(value); const json = JSON.stringify(bridge); @@ -479,15 +555,120 @@ function prepareBridgeSet(input: { throw invalid('Bridge record exceeds the supported byte bound.'); } if (byId.has(bridge.id)) throw invalid('Bridge set contains a duplicate identity.'); + totalBytes += bytes; + if (totalBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum) { + throw invalid('Bridge set exceeds the supported aggregate byte bound.'); + } byId.set(bridge.id, {bridge, bytes, digest: sha256HexSync(json), json}); } const bridges = [...byId.values()].sort((left, right) => compareBridges(left.bridge, right.bridge)); - return { - bridges, - digest: sha256HexSync( - ['threadnote-cross-repository-bridge-set-v1', ...bridges.map(entry => entry.json)].join('\n'), + const digest = new Bun.CryptoHasher('sha256'); + digest.update(BRIDGE_SET_DIGEST_DOMAIN); + for (const entry of bridges) { + digest.update('\n'); + digest.update(entry.json); + } + return {bridges, digest: digest.digest('hex'), totalBytes}; +} + +/** + * Conservative WAL/database headroom for replacing one bounded bridge set. + * Counts account for normalized columns and endpoint indexes without reading + * payloads into memory; byte totals account for both the old and new JSON. + */ +export function codeGraphCrossRepositoryBridgeReplacementRequiredFreeBytes(input: { + readonly existingBridgeBytes: number; + readonly existingBridgeCount: number; + readonly replacementBridgeBytes: number; + readonly replacementBridgeCount: number; +}): number { + for (const [label, value] of Object.entries(input)) { + if (!Number.isSafeInteger(value) || value < 0) throw invalid(`Bridge ${label} is invalid.`); + } + if ( + input.existingBridgeBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum || + input.replacementBridgeBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum || + input.replacementBridgeCount > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration || + input.existingBridgeCount > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration + ) { + throw invalid('Bridge replacement footprint exceeds the supported bound.'); + } + const rows = input.existingBridgeCount + input.replacementBridgeCount; + const logicalBytes = + input.existingBridgeBytes + input.replacementBridgeBytes + rows * BRIDGE_ROW_STORAGE_OVERHEAD_BYTES; + const safetyBytes = Math.max(BRIDGE_SET_DISK_SAFETY_BYTES, Math.ceil(logicalBytes * 0.1)); + const requiredBytes = logicalBytes * BRIDGE_SET_WRITE_AMPLIFICATION + safetyBytes; + if (!Number.isSafeInteger(requiredBytes) || requiredBytes < BRIDGE_SET_DISK_SAFETY_BYTES) { + throw invalid('Bridge replacement storage requirement exceeds the supported byte range.'); + } + return requiredBytes; +} + +function loadStoredBridgeFootprint(sql: SqlClient.SqlClient, generationId: string) { + return sql + .unsafe<{readonly bridge_count: unknown; readonly bridge_bytes: unknown}>( + `SELECT bridge_count, bridge_bytes FROM cross_repository_bridge_sets + WHERE generation_id = ? LIMIT 1`, + [generationId], + ) + .pipe( + Effect.flatMap(rows => + validateStored(() => { + if (rows.length === 0) return {bridgeCount: 0, totalBytes: 0} satisfies StoredBridgeFootprint; + if (rows.length !== 1) throw corrupt('Stored bridge footprint query returned an invalid row set.'); + const bridgeCount = requiredInteger(rows[0]!.bridge_count, 'stored bridge count'); + const totalBytes = requiredInteger(rows[0]!.bridge_bytes, 'stored bridge byte count'); + if ( + bridgeCount > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration || + totalBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum + ) { + throw corrupt('Stored bridge footprint exceeds the supported bound.'); + } + return {bridgeCount, totalBytes} satisfies StoredBridgeFootprint; + }), + ), + ); +} + +function verifyBridgeReplacementDiskCapacity( + probe: (target: string) => Effect.Effect, + target: string, + requiredBytes: number, +) { + return probe(target).pipe( + Effect.mapError( + cause => + new CodeGraphWorksetCatalogError( + 'storage', + `Could not inspect free disk space before bridge publication. Verify at least ${String(requiredBytes)} bytes are free and retry; the catalog was not modified.`, + {cause}, + ), ), - }; + Effect.flatMap(availableBytes => { + if (availableBytes === undefined) { + return Effect.fail( + new CodeGraphWorksetCatalogError( + 'storage', + `Could not determine free disk space before bridge publication. Verify at least ${String(requiredBytes)} bytes are free and retry; the catalog was not modified.`, + ), + ); + } + if (!Number.isSafeInteger(availableBytes) || availableBytes < 0) { + return Effect.fail( + new CodeGraphWorksetCatalogError('storage', 'The free disk space probe returned an invalid result.'), + ); + } + if (availableBytes < requiredBytes) { + return Effect.fail( + new CodeGraphWorksetCatalogError( + 'capacity', + `Bridge publication needs ${String(requiredBytes)} bytes free, but only ${String(availableBytes)} bytes are available. Free disk space and retry; the catalog was not modified.`, + ), + ); + } + return Effect.void; + }), + ); } function loadWritableGeneration(sql: SqlClient.SqlClient, generationId: string) { @@ -629,7 +810,7 @@ function loadPublishedBridgeSet(sql: SqlClient.SqlClient, generationId: string) return sql .unsafe( `SELECT s.generation_id, s.resolver_version, s.bridge_count, - s.bridge_set_digest, s.coverage_state, s.repository_count, + s.bridge_bytes, s.bridge_set_digest, s.coverage_state, s.repository_count, s.repositories_read, s.failed_repository_count, s.rejection_count, s.diagnostic_codes_json, g.workset_name FROM cross_repository_bridge_sets AS s @@ -648,12 +829,14 @@ function loadPublishedBridgeSet(sql: SqlClient.SqlClient, generationId: string) const row = rows[0]!; const resolverVersion = requiredInteger(row.resolver_version, 'bridge resolver version'); const bridgeCount = requiredInteger(row.bridge_count, 'bridge count'); + const bridgeBytes = requiredInteger(row.bridge_bytes, 'bridge byte count'); const digest = requiredText(row.bridge_set_digest, 'bridge-set digest'); const id = requiredText(row.generation_id, 'bridge generation identity'); const coverage = decodeCoverage(row); if ( resolverVersion !== CODE_GRAPH_CROSS_REPOSITORY_RESOLVER_VERSION || bridgeCount > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration || + bridgeBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum || !SHA256_HEX.test(digest) || !GENERATION_ID.test(id) ) { diff --git a/src/code_graph/deep_diagnostics.ts b/src/code_graph/deep_diagnostics.ts index 6f91b8ae..f148ba8d 100644 --- a/src/code_graph/deep_diagnostics.ts +++ b/src/code_graph/deep_diagnostics.ts @@ -5,6 +5,10 @@ import {CODE_GRAPH_DEEP_DIAGNOSTICS_WORKER_ARGUMENT} from '../worker_protocol.js import {type CodeGraphDatabaseHealth} from './store_models.js'; import {diagnoseCodeGraphDatabaseReadOnly} from './store_health.js'; +class CodeGraphDeepDiagnosticsError extends Error { + readonly _tag = 'CodeGraphDeepDiagnosticsError' as const; +} + const CODE_GRAPH_DEEP_DIAGNOSTICS_PROTOCOL = 1; const CODE_GRAPH_DEEP_DIAGNOSTICS_INPUT_BYTES_MAXIMUM = 64 * 1_024; const CODE_GRAPH_DEEP_DIAGNOSTICS_OUTPUT_BYTES_MAXIMUM = 4 * 1_024; @@ -43,7 +47,7 @@ export const diagnoseCodeGraphDatabaseDeepIsolated: ( }); const response = decodeDeepDiagnosticsResponse(result.stdout); if (response === undefined || !response.ok) { - return yield* Effect.fail(new Error('Isolated code graph deep diagnostics failed.')); + return yield* Effect.fail(new CodeGraphDeepDiagnosticsError('Isolated code graph deep diagnostics failed.')); } return response.health; }); @@ -92,7 +96,9 @@ function readBoundedWorkerInput(stdio: Stdio.Stdio): Effect.Effect { const size = state.size + encoder.encode(chunk).byteLength; if (size > CODE_GRAPH_DEEP_DIAGNOSTICS_INPUT_BYTES_MAXIMUM) { - return Effect.fail(new Error('Code graph deep diagnostics request exceeded its input limit.')); + return Effect.fail( + new CodeGraphDeepDiagnosticsError('Code graph deep diagnostics request exceeded its input limit.'), + ); } state.chunks.push(chunk); return Effect.succeed({chunks: state.chunks, size}); diff --git a/src/code_graph/disk_reservation.ts b/src/code_graph/disk_reservation.ts index 52611df5..013b2472 100644 --- a/src/code_graph/disk_reservation.ts +++ b/src/code_graph/disk_reservation.ts @@ -711,7 +711,7 @@ function classifyOwner(system: SystemInfoShape, processId: number) { return validProcessStartIdentity(processStartIdentity) ? ({processStartIdentity, state: 'running'} as const) : ({state: 'unknown'} as const); - }).pipe(Effect.catch(() => Effect.succeed({state: 'unknown'} as const))); + }); } function canonicalProcessStartIdentity(system: SystemInfoShape, processId: number) { diff --git a/src/code_graph/embedding.ts b/src/code_graph/embedding.ts index 6eb5cdcd..527d50c4 100644 --- a/src/code_graph/embedding.ts +++ b/src/code_graph/embedding.ts @@ -25,6 +25,10 @@ import { requireCodeGraphVectorRetirementSchema, } from './vector_retirement.js'; +class CodeGraphEmbeddingError extends Error { + readonly _tag = 'CodeGraphEmbeddingError' as const; +} + const CODE_GRAPH_EMBEDDING_TEMPLATE_VERSION = 1; const CODE_GRAPH_VECTOR_DATABASE_VERSION = 2; const CODE_GRAPH_SEMANTIC_MINIMUM_SCORE = 0.64; @@ -296,7 +300,10 @@ const ensureGraphVectors = Effect.fn('codeGraph.ensureVectors')(function* (input active.dimensions === selected.manifest.dimensions ? active : yield* selectMostRecentCompatibleGeneration(sql, selected.manifest.sha256, selected.manifest.dimensions!); - const generation = `${yield* Clock.currentTimeMillis}-${worktreeId.slice(-8)}-${input.snapshot.id.slice(-8)}`; + const generation = `${yield* Clock.currentTimeMillis}-${worktreeId.slice(-8)}-${input.snapshot.id.slice(-8)}-${(yield* crypto.randomUUIDv4).slice( + 0, + 8, + )}`; yield* sql` INSERT INTO vector_generations ( generation, snapshot_id, model_id, model_sha256, dimensions, @@ -530,10 +537,10 @@ const selectedEmbeddingModel = Effect.fn('codeGraph.selectedEmbeddingModel')(fun ) { const selection = yield* readModelSelection(threadnoteHome); const modelId = selection.roles.embedding; - if (!modelId) return yield* Effect.fail(new Error('No core embedding model is selected.')); + if (!modelId) return yield* Effect.fail(new CodeGraphEmbeddingError('No core embedding model is selected.')); const manifest = yield* catalog.get(modelId); if (manifest.role !== 'embedding' || !manifest.dimensions) { - return yield* Effect.fail(new Error(`Selected model ${modelId} is not an embedding model.`)); + return yield* Effect.fail(new CodeGraphEmbeddingError(`Selected model ${modelId} is not an embedding model.`)); } const verified = yield* store.verify(threadnoteHome, manifest); return {manifest, modelPath: verified.path}; @@ -629,12 +636,7 @@ const loadReusableVectors = Effect.fn('codeGraph.loadReusableVectors')(function* for (const row of rows) { if (expected.get(row.symbol_id) !== row.fingerprint) continue; const bytes = bytesFromSqlBlob(row.vector); - try { - decodeVector(bytes, dimensions); - reusable.set(row.symbol_id, bytes); - } catch { - // Corrupt reusable rows are re-embedded into the new atomic generation. - } + if (isDecodableVector(bytes, dimensions)) reusable.set(row.symbol_id, bytes); } return reusable; }); @@ -654,12 +656,12 @@ function insertVectorRows(sql: SqlClient.SqlClient, rows: readonly (readonly [st function encodeVector(vector: readonly number[], dimensions: number): Uint8Array { if (vector.length !== dimensions) { - throw new Error(`Vector has ${vector.length} dimensions; expected ${dimensions}.`); + throw new CodeGraphEmbeddingError(`Vector has ${vector.length} dimensions; expected ${dimensions}.`); } const bytes = new Uint8Array(dimensions * 4); const view = new DataView(bytes.buffer); for (const [index, component] of vector.entries()) { - if (!Number.isFinite(component)) throw new Error('Vector contains a non-finite component.'); + if (!Number.isFinite(component)) throw new CodeGraphEmbeddingError('Vector contains a non-finite component.'); view.setFloat32(index * 4, component, true); } return bytes; @@ -668,25 +670,34 @@ function encodeVector(vector: readonly number[], dimensions: number): Uint8Array function decodeVector(value: unknown, dimensions: number): readonly number[] { const bytes = bytesFromSqlBlob(value); if (bytes.byteLength !== dimensions * 4) { - throw new Error(`Stored vector has ${bytes.byteLength} bytes; expected ${dimensions * 4}.`); + throw new CodeGraphEmbeddingError(`Stored vector has ${bytes.byteLength} bytes; expected ${dimensions * 4}.`); } const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const vector = Array.from({length: dimensions}, (_, index) => view.getFloat32(index * 4, true)); if (vector.some(component => !Number.isFinite(component))) { - throw new Error('Stored vector contains a non-finite component.'); + throw new CodeGraphEmbeddingError('Stored vector contains a non-finite component.'); } const magnitude = Math.sqrt(vector.reduce((sum, component) => sum + component * component, 0)); - if (Math.abs(magnitude - 1) > 0.002) throw new Error('Stored vector is not L2-normalized.'); + if (Math.abs(magnitude - 1) > 0.002) throw new CodeGraphEmbeddingError('Stored vector is not L2-normalized.'); return vector; } +function isDecodableVector(value: unknown, dimensions: number): boolean { + try { + decodeVector(value, dimensions); + return true; + } catch { + return false; + } +} + function bytesFromSqlBlob(value: unknown): Uint8Array { if (value instanceof Uint8Array) return value; if (value instanceof ArrayBuffer) return new Uint8Array(value); if (ArrayBuffer.isView(value)) { return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } - throw new Error('Stored vector is not a binary SQLite value.'); + throw new CodeGraphEmbeddingError('Stored vector is not a binary SQLite value.'); } function mergeSearchResults( @@ -706,8 +717,10 @@ function useVectorDatabase( effect: Effect.Effect, ): Effect.Effect> { return Effect.scoped( - effect.pipe(Effect.provide(SqliteClient.layer({disableWAL: true, filename: databasePath}))), - ) as Effect.Effect>; + Layer.build(SqliteClient.layer({disableWAL: true, filename: databasePath})).pipe( + Effect.flatMap(context => effect.pipe(Effect.provide(context))), + ), + ); } const initializeVectorDatabase = Effect.fn('codeGraph.initializeVectorDatabase')(function* (sql: SqlClient.SqlClient) { @@ -721,7 +734,7 @@ const initializeVectorDatabase = Effect.fn('codeGraph.initializeVectorDatabase') return; } if (version !== 0) { - return yield* Effect.fail(new Error('Code graph vector database version is unsupported.')); + return yield* Effect.fail(new CodeGraphEmbeddingError('Code graph vector database version is unsupported.')); } const objects = yield* sql.unsafe( `SELECT 1 FROM sqlite_master @@ -729,7 +742,7 @@ const initializeVectorDatabase = Effect.fn('codeGraph.initializeVectorDatabase') LIMIT 1`, ); if (objects.length !== 0) { - return yield* Effect.fail(new Error('Code graph vector database initialization is incomplete.')); + return yield* Effect.fail(new CodeGraphEmbeddingError('Code graph vector database initialization is incomplete.')); } yield* sql.withTransaction( Effect.gen(function* () { @@ -750,7 +763,7 @@ const requireVectorDatabaseReady = Effect.fn('codeGraph.requireVectorDatabaseRea ) { const versions = yield* sql.unsafe<{readonly user_version: unknown}>('PRAGMA user_version'); if (versions.length !== 1 || versions[0]?.user_version !== CODE_GRAPH_VECTOR_DATABASE_VERSION) { - return yield* Effect.fail(new Error('Code graph vector database version is unsupported.')); + return yield* Effect.fail(new CodeGraphEmbeddingError('Code graph vector database version is unsupported.')); } yield* requireCodeGraphVectorRetirementSchema(sql); }); @@ -837,7 +850,8 @@ const removeLegacyVectorSidecars = Effect.fn('codeGraph.removeLegacyVectorSideca }); function requiredWorktreeId(layout: CodeGraphLayout): string { - if (!/^[0-9a-f]{64}$/.test(layout.worktreeId)) throw new Error('Code graph worktree identity is invalid.'); + if (!/^[0-9a-f]{64}$/.test(layout.worktreeId)) + throw new CodeGraphEmbeddingError('Code graph worktree identity is invalid.'); return layout.worktreeId; } diff --git a/src/code_graph/git_worktree_registration_worker.ts b/src/code_graph/git_worktree_registration_worker.ts index f9603b52..8417af27 100644 --- a/src/code_graph/git_worktree_registration_worker.ts +++ b/src/code_graph/git_worktree_registration_worker.ts @@ -1,5 +1,5 @@ import {Effect, FileSystem, Stdio, Stream} from 'effect'; -import {fromPromiseError} from '../effect/errors.js'; +import {fromPromise} from '../effect/errors.js'; import {SystemInfo} from '../effect/system.js'; import { CODE_GRAPH_GIT_WORKTREE_REGISTRATION_LIMITS, @@ -11,6 +11,10 @@ import { type CodeGraphGitWorktreeRegistryRequest, } from './git_worktree_registration.js'; +class GitWorktreeRegistrationError extends Error { + readonly _tag = 'GitWorktreeRegistrationError' as const; +} + const UTF8 = new TextEncoder(); /** One-shot, path-private helper. It always emits one bounded, path-free response. */ @@ -29,23 +33,23 @@ export const gitWorktreeRegistrationWorkerProgram = Effect.gen(function* () { const workerResponse = Effect.fn('codeGraph.gitWorktreeRegistrationWorkerResponse')(function* (input: Uint8Array) { const decoded = yield* Effect.try({ try: () => new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}).decode(input), - catch: () => new Error('invalid'), + catch: () => new GitWorktreeRegistrationError('invalid'), }); if (!decoded.endsWith('\n') || decoded.slice(0, -1).includes('\n')) { - return yield* Effect.fail(new Error('invalid')); + return yield* Effect.fail(new GitWorktreeRegistrationError('invalid')); } const request = yield* Effect.try({ try: (): unknown => JSON.parse(decoded.slice(0, -1)), - catch: () => new Error('invalid'), + catch: () => new GitWorktreeRegistrationError('invalid'), }); if (validCodeGraphWorktreeAuthorityWorkerRequest(request)) { yield* blockAuthorityLstatForTest(); - return yield* fromPromiseError(() => scanCodeGraphWorktreeAuthorityWorkerRequest(request)); + return yield* fromPromise('scan worktree authority', () => scanCodeGraphWorktreeAuthorityWorkerRequest(request)); } if (validCodeGraphGitWorktreeRegistryBatchRequest(request)) { - return yield* fromPromiseError(() => scanCodeGraphGitWorktreeRegistryBatch(request)); + return yield* fromPromise('scan Git worktree registry', () => scanCodeGraphGitWorktreeRegistryBatch(request)); } - return yield* fromPromiseError(() => + return yield* fromPromise('scan Git worktree registration batch', () => scanCodeGraphGitWorktreeRegistry(request as CodeGraphGitWorktreeRegistryRequest), ); }); @@ -59,7 +63,7 @@ const readBoundedStandardInput = Effect.fn('codeGraph.readGitWorktreeRegistratio yield* stdio.stdin.pipe( Stream.runForEach(chunk => { total += chunk.byteLength; - if (total > limit) return Effect.fail(new Error('invalid')); + if (total > limit) return Effect.fail(new GitWorktreeRegistrationError('invalid')); chunks.push(chunk); return Effect.void; }), diff --git a/src/code_graph/indexer.ts b/src/code_graph/indexer.ts index 345057f9..b98a411d 100644 --- a/src/code_graph/indexer.ts +++ b/src/code_graph/indexer.ts @@ -1,134 +1,3 @@ -import {Clock, Context, Crypto, Effect, FileSystem, Layer, Option, Path} from 'effect'; -import {sha256HexSync} from '../crypto/sha256.js'; -import {CommandExecutor} from '../effect/command.js'; -import {withExclusiveFileLock} from '../effect/file_lock.js'; -import {SystemInfo, type SystemInfoShape} from '../effect/system.js'; -import {withThreadnoteProcessActivity} from '../process_diagnostics.js'; -import {codeGraphBlobExtractionReuseClass, codeGraphBlobReuseCacheKey} from './blob_reuse.js'; -import type {CodeGraphBuildOwnerIdentity} from './build_owner.js'; -import {CODE_GRAPH_CACHE_TRANSACTION_LIMITS, codeGraphFileBlobCapacityBytes} from './cache_capacity.js'; -import {createRepositoryFactAttributor, extractRepositoryFileFacts} from './extractor.js'; -import {planCodeGraphExtractionLanes} from './extraction_lanes.js'; -import { - budgetCachedCodeGraphFacts, - cachedCodeGraphFactBytes, - CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM, - finalCodeGraphFactBatches, - serializeBoundedCodeGraphFact, - type BoundedCodeGraphFact, -} from './fact_budget.js'; -import { - assessProjectClosureSeeds, - planProjectIncrementalClosure, - PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES, - PROJECT_INCREMENTAL_CLOSURE_MAX_FILES, - PROJECT_INCREMENTAL_CLOSURE_MAX_SOURCE_BYTES, - selectProjectIncrementalClosure, -} from './incremental_closure.js'; -import {preferredIncrementalBaseCommitGroups} from './incremental_base_selection.js'; -import { - codeGraphIncrementalWorkFitsBudget, - measureCodeGraphIncrementalWork, - type CodeGraphIncrementalWork, -} from './incremental_work.js'; -import { - inventoryRepository, - worktreeBuildRequestState, - type CodeGraphContentBatchContext, - type CodeGraphInventoryOptions, -} from './inventory.js'; -import { - BUILTIN_LANGUAGE_PACK_REGISTRY, - CodeGraphLanguagePackRegistry, - packDerivationIdentity, - type CodeGraphLanguagePackRegistryShape, -} from './languages/registry.js'; -import {assessCodeGraphLanguagePackDelta} from './languages/provenance.js'; -import { - codeGraphDiskReservationLockPath, - codeGraphDiskReservationRoot, - codeGraphLayout, - codeGraphRequestBuildLockPath, - codeGraphSnapshotBuildLockPath, -} from './layout.js'; -import {CodeGraphMaintenanceCoordinator, type CodeGraphMaintenanceCoordinatorShape} from './maintenance_coordinator.js'; -import {runCodeGraphLifecycleOpportunity} from './lifecycle_opportunity.js'; -import {codeGraphMaintenanceIntentActive, withCodeGraphMaintenanceRegistration} from './maintenance_gate.js'; -import {resolveAndRecordCodeGraphLocalAssociation} from './local_provenance.js'; -import {compareCodeUnits} from './ordering.js'; -import {canonicalCodeGraphMonikers} from './cross_repository/monikers.js'; -import type {CodeGraphMonikerV1} from './cross_repository/types.js'; -import { - codeGraphExtractionWorkUnits, - codeGraphSourceSizeBucket, - type CodeGraphScanningMetrics, -} from './progress_telemetry.js'; -import {relocateStructuredSchemaFacts} from './languages/schemas/extractor.js'; -import {repositoryIdentityMatchesExpectation, resolveRepositoryIdentity} from './repository.js'; -import { - codeGraphDiskCapacityFailure, - codeGraphPersistentCapacityDemand, - isCodeGraphCapacityPause, - type CodeGraphDirectPersistentCapacityBoundary, -} from './disk_capacity.js'; -import { - codeGraphDiskReservationFilesystemKey, - type CodeGraphDiskReservationOptions, - withCodeGraphDiskReservation, -} from './disk_reservation.js'; -import { - CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION, - CODE_GRAPH_REUSABLE_BASE_RECEIPT_VERSION, - CodeGraphStore, - materializedShardDerivationIdentity, - type CodeGraphRetiredSnapshotCleanupProgress, - type CodeGraphDirectPersistentCapacityProtector, - type CodeGraphReusableCleanBase, - type CodeGraphLanguagePackProvenance, - type CodeGraphReusableReexport, - type CodeGraphReusableReexportSeed, - type CodeGraphStagingProgress, - type CodeGraphStoreShape, - type CodeGraphSqliteWriterSettings, - type CodeGraphSqliteWriterTuning, -} from './store.js'; -import {inspectCodeGraphStorage} from './storage.js'; -import { - CODE_GRAPH_EXTRACTOR_SET_VERSION, - type CodeGraphEdge, - type CodeGraphFileFacts, - type CodeGraphIndexSummary, - type CodeGraphInventoryFile, - type CodeGraphMaterializationActivity, - type CodeGraphMaterializationMetrics, - type CodeGraphMaterializationRows, - type CodeGraphOverlayFallbackReason, - type CodeGraphProgress, - type CodeGraphReference, - type CodeGraphSnapshot, - type CodeGraphSymbol, - type RepositoryIdentity, - type RepositoryIdentityExpectation, -} from './types.js'; -import type {CodeGraphInventory} from './inventory.js'; -import type {CodeGraphLayout} from './layout.js'; -import { - CodeGraphEmbeddingIndex, - type CodeGraphEmbeddingIndexShape, - type CodeGraphEmbeddingStatus, -} from './embedding.js'; -import {TreeSitterRuntime, type TreeSitterRuntimeShape} from './tree_sitter/runtime.js'; -import {createWorkspaceAttributor} from './workspace.js'; -import {assessCodeGraphWorkspaceCompatibility} from './workspace_compatibility.js'; -import {makeCodeGraphBuildReporter, readCodeGraphBuildStatuses} from './build_status.js'; -import type {CodeGraphWorkspace} from './languages/types.js'; -import { - budgetParserWorkerFacts, - CodeGraphParserPool, - type CodeGraphParserPoolShape, - type CodeGraphParserResult, -} from './parser_worker.js'; - export { budgetCachedCodeGraphFacts, cachedCodeGraphFactBytes, @@ -140,4975 +9,45 @@ export { finalCodeGraphFactBatches, } from './fact_budget.js'; -export interface CodeGraphIndexOptions extends CodeGraphInventoryOptions { - readonly cwd: string; - /** When false, skip blocking vector materialization after a ready structural snapshot. */ - readonly ensureVectors?: boolean; - /** Exact graph target supplied by a trusted local administration surface. */ - readonly expectedIdentity?: RepositoryIdentityExpectation; - readonly force?: boolean; - /** Internal benchmark/correctness escape hatch; normal indexing keeps this enabled. */ - readonly incrementalOverlay?: boolean; - /** @internal Records read-back PRAGMA values for controlled benchmark evidence. */ - readonly onSqliteWriterConfigured?: (settings: CodeGraphSqliteWriterSettings) => Effect.Effect; - /** @internal Benchmark-only physical transaction grouping; normal indexing uses four logical receipts. */ - readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; - /** @internal Benchmark-only SQLite writer candidate; normal indexing leaves this unset. */ - readonly sqliteWriterTuning?: CodeGraphSqliteWriterTuning; - /** @internal Deterministic fresh-capacity probe used by lifecycle fault tests. */ - readonly diskCapacityAvailableBytes?: ( - path: string, - boundary: CodeGraphDirectPersistentCapacityBoundary, - ) => Effect.Effect; - readonly threadnoteHome: string; -} - -export interface DirectPersistentCapacityProtection { - readonly availableDiskBytes: ( - path: string, - boundary: CodeGraphDirectPersistentCapacityBoundary, - ) => Effect.Effect; - readonly crypto: Crypto.Crypto; - readonly maintenance: CodeGraphMaintenanceCoordinatorShape; - readonly path: Path.Path; - readonly system: SystemInfoShape; - readonly temporaryDirectory: string; - readonly walAutoCheckpointPages: number; -} - -export function codeGraphIndexEnsuresVectors(options: {readonly ensureVectors?: boolean}): boolean { - return options.ensureVectors !== false; -} - -interface CommittedBaseResult { - readonly diagnostics: readonly string[]; - readonly leaseToken: Option.Option; - readonly snapshot: CodeGraphSnapshot; - readonly stagingReusable: boolean; -} - -type IncrementalOverlayAssessment = - | { - readonly facts: readonly CodeGraphFileFacts[]; - readonly files: readonly CodeGraphInventoryFile[]; - readonly closureProjects?: number; - readonly mode: 'eligible'; - readonly deletedPaths?: readonly string[]; - readonly resolutionClosure?: 'changed' | 'full' | 'project'; - readonly extractorTransition?: true; - readonly reuse: 'persisted-base' | 'staged-base'; - readonly work: CodeGraphIncrementalWork; - } - | { - readonly mode: 'fallback'; - readonly reason: CodeGraphOverlayFallbackReason; - }; - -type IncrementalOverlayPreassessment = - | { - readonly committedWorkspace: CodeGraphWorkspace; - readonly facts: readonly CodeGraphFileFacts[]; - readonly files: readonly CodeGraphInventoryFile[]; - readonly closureProjects?: number; - readonly mode: 'compatible'; - readonly deletedPaths?: readonly string[]; - readonly resolutionClosure?: 'changed' | 'full' | 'project'; - readonly extractorTransition?: true; - } - | { - readonly mode: 'fallback'; - readonly reason: CodeGraphOverlayFallbackReason; - }; - -type ReusableCleanSnapshotAttempt = - | { - readonly mode: 'complete'; - readonly summary: CodeGraphIndexSummary; - } - | { - readonly mode: 'fallback'; - readonly reason: CodeGraphOverlayFallbackReason; - }; - -export interface CodeGraphCommitLease { - readonly leaseToken: string; - readonly snapshot: CodeGraphSnapshot; -} - -export interface CodeGraphIndexerShape { - readonly ensureCommit: ( - options: Omit & {readonly commit: string}, - ) => Effect.Effect; - readonly index: (options: CodeGraphIndexOptions) => Effect.Effect; -} - -export class CodeGraphIndexer extends Context.Service()( - 'threadnote/codeGraph/CodeGraphIndexer', -) { - static readonly layer = Layer.effect( - CodeGraphIndexer, - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const store = yield* CodeGraphStore; - const maintenance = yield* CodeGraphMaintenanceCoordinator; - const embedding = yield* CodeGraphEmbeddingIndex; - const languagePacks = yield* CodeGraphLanguagePackRegistry; - const treeSitter = yield* TreeSitterRuntime; - const parserPool = yield* CodeGraphParserPool; - const command = yield* CommandExecutor; - const crypto = yield* Crypto.Crypto; - const system = yield* SystemInfo; - const index = (request: CodeGraphIndexOptions, attempt = 0): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const initialIdentity = yield* resolveRepositoryIdentity(request.cwd); - if ( - request.expectedIdentity && - !repositoryIdentityMatchesExpectation(initialIdentity, request.expectedIdentity) - ) { - return yield* Effect.fail(new Error('Repository identity does not match the requested graph target.')); - } - const layout = codeGraphLayout( - path, - request.threadnoteHome, - initialIdentity.checkoutId, - initialIdentity.worktreeId, - ); - const requestedOverlay = yield* worktreeBuildRequestState(initialIdentity, request.threadnoteHome); - const requestKey = request.force - ? undefined - : codeGraphBuildRequestKey(initialIdentity, requestedOverlay, languagePacks, request.incrementalOverlay); - const reporter = yield* withCodeGraphMaintenanceRegistration( - request.threadnoteHome, - Effect.gen(function* () { - if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph repository root is a symbolic link.')); - } - yield* fs.makeDirectory(layout.repositoryRoot, {recursive: true, mode: 0o700}); - const reporter = yield* makeCodeGraphBuildReporter( - initialIdentity, - layout, - requestKey ? {key: requestKey} : undefined, - ); - yield* request.onProgress?.({phase: 'registering'}) ?? Effect.void; - return reporter; - }), - ); - yield* Effect.forkScoped(reporter.heartbeat); - const options: CodeGraphIndexOptions = { - ...request, - onProgress: progress => - reporter.progress(progress).pipe(Effect.andThen(request.onProgress?.(progress) ?? Effect.void)), - }; - const capacityProtection: DirectPersistentCapacityProtection = { - availableDiskBytes: - options.diskCapacityAvailableBytes ?? ((target: string) => system.availableDiskBytes(target)), - crypto, - maintenance, - path, - system, - temporaryDirectory: system.tempDirectory, - walAutoCheckpointPages: options.sqliteWriterTuning?.walAutoCheckpointPages ?? 1_000, - }; - const ensureVectors = codeGraphIndexEnsuresVectors(options); - const summary = yield* withCodeGraphProcessLock( - fs, - layout.lockPath, - () => - (options.onProgress?.({phase: 'waiting', reason: 'repository-lock'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - 'index-repository', - Effect.gen(function* () { - if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph repository root is a symbolic link.')); - } - if (!(yield* fs.exists(layout.repositoryRoot))) { - return yield* Effect.fail(new RepositoryRegistrationLost()); - } - if (yield* codeGraphMaintenanceIntentActive(options.threadnoteHome)) { - return yield* Effect.fail(new RepositoryMaintenanceInterrupted()); - } - const build = store - .withSession( - layout.databasePath, - Effect.gen(function* () { - const startedAt = yield* Clock.currentTimeMillis; - const {identity} = yield* resolveAndRecordCodeGraphLocalAssociation( - options.threadnoteHome, - options.cwd, - { - validateIdentity: identity => { - if (!repositoryIdentityMatchesExpectation(identity, initialIdentity)) { - return Effect.fail( - new Error('Repository identity changed while waiting for the graph lock.'), - ); - } - if ( - options.expectedIdentity && - !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity) - ) { - return Effect.fail( - new Error('Repository identity does not match the requested graph target.'), - ); - } - return identity.headCommit === initialIdentity.headCommit - ? Effect.void - : Effect.fail(new WorktreeChangedDuringIndex()); - }, - }, - ); - yield* store.initialize(layout.databasePath); - { - const currentOverlay = yield* worktreeBuildRequestState(identity, options.threadnoteHome); - if (!sameOverlayState(currentOverlay, requestedOverlay)) { - return yield* Effect.fail(new WorktreeChangedDuringIndex()); - } - if (requestKey) { - const completedByOwner = yield* completedConcurrentSnapshot( - store, - layout, - identity, - currentOverlay, - requestKey, - options.incrementalOverlay === false, - ); - if (completedByOwner) { - yield* store.retireIncompleteWorktreeSnapshots( - layout.databasePath, - identity.repositoryId, - identity.worktreeId, - new Set(), - retiredSnapshotCleanupReporter(options.onProgress), - {cleanupMode: 'deferred'}, - ); - yield* promoteReadySnapshotWithCapacity( - { - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - store, - threadnoteHome: options.threadnoteHome, - }, - completedByOwner.id, - ); - return yield* reuseReadySnapshot({ - embedding, - ensureVectors, - identity, - layout, - onProgress: options.onProgress, - reusedFiles: completedByOwner.fileCount, - skippedFiles: 0, - snapshot: completedByOwner, - startedAt, - store, - threadnoteHome: options.threadnoteHome, - totalFiles: completedByOwner.fileCount, - }); - } - } - } - const cachedCommittedFileKeys = options.force - ? new Set() - : yield* cachedFileKeys(store, layout.databasePath, languagePacks); - const cacheCoalescer = cacheContentBatch({ - databasePath: layout.databasePath, - languagePacks, - onProgress: options.onProgress, - parserPool, - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - threadnoteHome: options.threadnoteHome, - }), - store, - threadnoteHome: options.threadnoteHome, - treeSitter, - }); - const inventory = yield* inventoryRepository(identity, { - ...options, - cachedCommittedFileKeys, - includeOpaqueCorpusAssets: ensureVectors, - languagePacks, - onContentBatch: cacheCoalescer.onContentBatch, - }).pipe( - Effect.tap(() => cacheCoalescer.flush()), - Effect.ensuring(cacheCoalescer.discard().pipe(Effect.andThen(parserPool.trimIdle()))), - ); - // Inventory and extraction build large, short-lived maps and Git payloads. Reclaim them before - // the SQLite activation phase so their heap high-water does not overlap the writer page cache. - yield* Effect.sync(() => { - Bun.gc(true); - Bun.shrink(); - }); - yield* Effect.yieldNow; - const extractorSet = extractorSetIdentity(inventory.files, languagePacks); - const graphContentId = graphContentIdentity(extractorSet, inventory.files); - const logicalSnapshotId = snapshotIdentity( - identity, - inventory.dirty, - extractorSet, - inventory.files, - ); - const forceGeneration = options.force - ? (yield* crypto.randomUUIDv4).replaceAll('-', '').slice(0, 16) - : undefined; - const forcedSnapshotId = forcedSnapshotIdentity(logicalSnapshotId, forceGeneration); - const directSnapshotId = directFullSnapshotIdentity(logicalSnapshotId); - const resumedForcedBuild = options.force - ? yield* store.resumableForcedBuild(layout.databasePath, logicalSnapshotId) - : undefined; - const readyCandidateIds = inventory.dirty - ? options.incrementalOverlay === false - ? [directSnapshotId] - : [logicalSnapshotId, directSnapshotId] - : [logicalSnapshotId]; - const existing = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); - const reusableExisting = existing - ? yield* store.currentLexicalReadySnapshotById(layout.databasePath, existing.id) - : undefined; - const reusableReadyById = !options.force - ? reusableExisting && readyCandidateIds.includes(reusableExisting.id) - ? reusableExisting - : yield* firstReadySnapshotById(store, layout.databasePath, readyCandidateIds) - : undefined; - // Exact cgsn_* can miss when inventory source/provenance differs slightly - // from the shared clean row while graph content is identical. Prefer promote - // of a HEAD-matching clean ready snapshot over rematerializing. - const reusableReady = - reusableReadyById ?? - (!options.force && !inventory.dirty - ? yield* reusableReadySnapshotForCleanCommit({ - databasePath: layout.databasePath, - extractorSet, - graphContentId, - headCommit: identity.headCommit, - repositoryId: identity.repositoryId, - store, - }) - : undefined); - // A ready candidate wins this request. Do not preserve an - // interrupted logical/direct sibling that cannot be used on - // the early-return path: a repository-sized persistent build - // would otherwise remain reachable forever unless the user - // explicitly selected that other materialization mode again. - const retainedSnapshotIds = reusableReady - ? new Set() - : options.force - ? new Set([resumedForcedBuild?.id ?? forcedSnapshotId]) - : inventory.dirty - ? new Set(readyCandidateIds) - : new Set([logicalSnapshotId]); - yield* store.retireIncompleteWorktreeSnapshots( - layout.databasePath, - identity.repositoryId, - identity.worktreeId, - retainedSnapshotIds, - retiredSnapshotCleanupReporter(options.onProgress), - {cleanupMode: 'deferred'}, - ); - if (reusableReady) { - if (existing?.id !== reusableReady.id) { - yield* promoteReadySnapshotWithCapacity( - { - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - store, - threadnoteHome: options.threadnoteHome, - }, - reusableReady.id, - ); - } - return yield* reuseReadySnapshot({ - embedding, - ensureVectors, - identity, - layout, - onProgress: options.onProgress, - reusedFiles: inventory.files.length - inventory.parsedFiles, - skippedFiles: inventory.skipped, - snapshot: reusableReady, - startedAt, - store, - threadnoteHome: options.threadnoteHome, - totalFiles: inventory.files.length, - }); - } - if (!inventory.dirty) { - return yield* buildOwnedCleanSnapshot({ - buildOwner: reporter.ownerIdentity, - capacityProtection, - embedding, - ensureVectors, - existing, - fallbackSnapshotId: forcedSnapshotId, - force: options.force === true, - fs, - identity, - inventory, - languagePacks, - layout, - logicalSnapshotId, - onProgress: options.onProgress, - persistentMaterializationTransactionBatchLimit: - options.persistentMaterializationTransactionBatchLimit, - requestedOverlay, - startedAt, - store, - threadnoteHome: options.threadnoteHome, - }); - } - const canAttemptIncrementalOverlay = - inventory.dirty && options.incrementalOverlay !== false && options.force !== true; - const resumableDirectBuild = - inventory.dirty && !options.force - ? yield* store.resumableBuildById(layout.databasePath, directSnapshotId) - : undefined; - let workspace = inventory.workspace ?? (yield* languagePacks.discoverWorkspace(inventory.files)); - let committedBase: CommittedBaseResult | undefined; - let incrementalAssessment: IncrementalOverlayAssessment | undefined; - let incrementalPrepared = false; - let building: CodeGraphSnapshot; - let persistentOwnerToken: string | undefined; - if (resumedForcedBuild) { - building = resumedForcedBuild; - incrementalAssessment = {mode: 'fallback', reason: 'forced-full-rebuild'}; - persistentOwnerToken = yield* store.claimPersistentBuild( - layout.databasePath, - identity, - building, - {logicalSnapshotId, owner: reporter.ownerIdentity}, - ); - } else if (resumableDirectBuild) { - building = resumableDirectBuild; - incrementalAssessment = { - mode: 'fallback', - reason: options.incrementalOverlay === false ? 'disabled' : 'staging-unavailable', - }; - persistentOwnerToken = yield* store.claimPersistentBuild( - layout.databasePath, - identity, - building, - {logicalSnapshotId, owner: reporter.ownerIdentity}, - ); - } else if (!inventory.dirty && !options.force) { - building = { - commit: identity.headCommit, - dirty: false, - edgeCount: 0, - extractorSet, - fileCount: 0, - graphContentId, - id: logicalSnapshotId, - repositoryId: identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: identity.worktreeId, - }; - persistentOwnerToken = yield* store.claimPersistentBuild( - layout.databasePath, - identity, - building, - {logicalSnapshotId, owner: reporter.ownerIdentity}, - ); - } else if (!canAttemptIncrementalOverlay) { - building = { - commit: identity.headCommit, - dirty: inventory.dirty, - edgeCount: 0, - extractorSet, - fileCount: 0, - graphContentId, - id: options.force ? forcedSnapshotId : directSnapshotId, - overlayFingerprint: inventory.overlayFingerprint, - repositoryId: identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: identity.worktreeId, - }; - incrementalAssessment = { - mode: 'fallback', - reason: options.force ? 'forced-full-rebuild' : 'disabled', - }; - persistentOwnerToken = yield* store.claimPersistentBuild( - layout.databasePath, - identity, - building, - {logicalSnapshotId, owner: reporter.ownerIdentity}, - ); - } else { - const reusableDirtyBase = yield* attemptReusableDirtyBase( - { - extractorSet, - identity, - inventory, - languagePacks, - layout, - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - threadnoteHome: options.threadnoteHome, - }), - store, - }, - workspace, - ); - let preassessment: IncrementalOverlayPreassessment; - let incrementalBuilding: CodeGraphSnapshot | undefined; - if (Option.isSome(reusableDirtyBase)) { - committedBase = reusableDirtyBase.value.committedBase; - preassessment = reusableDirtyBase.value.preassessment; - } else { - preassessment = yield* assessIncrementalOverlayCompatibility( - {extractorSet, inventory, languagePacks, layout, store}, - workspace, - ); - if (preassessment.mode === 'compatible') { - committedBase = yield* ensureCommittedBase({ - buildOwner: reporter.ownerIdentity, - capacityProtection, - embedding, - existing, - force: false, - forceGeneration, - fs, - identity, - inventory, - languagePacks, - layout, - onProgress: options.onProgress, - persistentMaterializationTransactionBatchLimit: - options.persistentMaterializationTransactionBatchLimit, - requestedOverlay, - startedAt, - store, - threadnoteHome: options.threadnoteHome, - }); - } - } - if (preassessment.mode === 'fallback') { - incrementalAssessment = preassessment; - } else { - incrementalBuilding = { - baseSnapshotId: committedBase!.snapshot.id, - commit: identity.headCommit, - dirty: inventory.dirty, - edgeCount: 0, - extractorSet, - fileCount: 0, - graphContentId, - id: logicalSnapshotId, - overlayFingerprint: inventory.overlayFingerprint, - repositoryId: identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: identity.worktreeId, - }; - incrementalAssessment = yield* assessIncrementalOverlay( - { - building: incrementalBuilding, - committedBase: committedBase!, - force: false, - incrementalOverlayEnabled: true, - inventory, - languagePacks, - layout, - store, - }, - workspace, - preassessment, - ); - } - if (incrementalAssessment.mode === 'eligible') { - if (committedBase === undefined) { - return yield* Effect.fail( - new Error('Incremental code graph preparation requires a committed base snapshot.'), - ); - } - const incrementalReusedFiles = inventory.files.length - incrementalAssessment.files.length; - const incrementalCapacityProtector = codeGraphDirectPersistentCapacityProtector({ - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - threadnoteHome: options.threadnoteHome, - }); - yield* options.onProgress?.({ - completed: 0, - phase: 'materializing', - reused: incrementalReusedFiles, - total: incrementalAssessment.files.length, - unit: 'files', - }) ?? Effect.void; - incrementalPrepared = - incrementalAssessment.reuse === 'persisted-base' - ? yield* store.preparePersistedIncrementalActivation( - layout.databasePath, - committedBase.snapshot.id, - incrementalAssessment.files, - incrementalAssessment.facts, - { - deletedPaths: incrementalAssessment.deletedPaths, - resolutionClosure: incrementalAssessment.resolutionClosure, - }, - incrementalCapacityProtector, - ) - : yield* store.replaceStagedModifiedFiles( - layout.databasePath, - committedBase.snapshot.id, - incrementalAssessment.files, - incrementalAssessment.facts, - incrementalCapacityProtector, - ); - if (!incrementalPrepared) { - incrementalAssessment = {mode: 'fallback', reason: 'staging-identity-mismatch'}; - } - } - if (incrementalPrepared && incrementalBuilding !== undefined) { - building = incrementalBuilding; - yield* store.markBuilding(layout.databasePath, identity, building); - } else { - building = { - commit: identity.headCommit, - dirty: inventory.dirty, - edgeCount: 0, - extractorSet, - fileCount: 0, - id: directSnapshotId, - overlayFingerprint: inventory.overlayFingerprint, - repositoryId: identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: identity.worktreeId, - }; - committedBase = undefined; - persistentOwnerToken = yield* store.claimPersistentBuild( - layout.databasePath, - identity, - building, - {logicalSnapshotId, owner: reporter.ownerIdentity}, - ); - } - } - if (incrementalPrepared) { - // The prepared delta already contains attributed facts - // and a staged workspace catalog. Retaining thousands - // of project/dependency objects through activation only - // makes one-file overlays overlap full-workspace memory - // with SQLite's effective-graph scans. - workspace = { - diagnostics: workspace.diagnostics, - fingerprint: workspace.fingerprint, - projects: [], - workspaces: [], - }; - } - return yield* buildAndActivate({ - activatePointer: true, - building, - capacityProtection, - existing, - embedding, - ensureVectors, - force: options.force === true, - fs, - identity, - inventory, - committedBase, - incrementalAssessment, - incrementalOverlayEnabled: options.incrementalOverlay !== false, - incrementalPrepared, - languagePacks, - layout, - onProgress: options.onProgress, - persistentMaterializationTransactionBatchLimit: - options.persistentMaterializationTransactionBatchLimit, - persistentOwnerToken, - requestedOverlay, - startedAt, - store, - threadnoteHome: options.threadnoteHome, - workspace, - }).pipe( - Effect.catch(cause => - persistentOwnerToken !== undefined && isCodeGraphCapacityPause(cause) - ? Effect.fail(cause) - : store - .markFailed(layout.databasePath, building.id, messageOf(cause), persistentOwnerToken) - .pipe(Effect.andThen(Effect.fail(cause))), - ), - ); - }), - writerSessionOptions(layout, options), - ) - .pipe( - Effect.tap(summary => reporter.complete(summary)), - Effect.tapError(cause => reporter.fail(cause)), - ); - return yield* withSharedCleanRequestGate({ - checkoutId: initialIdentity.checkoutId, - effect: build, - fs, - onProgress: options.onProgress, - path, - requestKey, - requestedOverlay, - threadnoteHome: options.threadnoteHome, - }); - }), - ).pipe( - Effect.ensuring( - runCodeGraphLifecycleOpportunity({ - maintenance, - opportunity: 'index-completion', - targets: [ - {anchorIdentity: initialIdentity, checkoutId: layout.checkoutId, databasePath: layout.databasePath}, - ], - threadnoteHome: request.threadnoteHome, - }).pipe(Effect.ignore), - ), - ); - return summary; - }), - ).pipe( - Effect.provideService(CommandExecutor, command), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(SystemInfo, system), - Effect.catchIf( - cause => cause instanceof WorktreeChangedDuringIndex && attempt === 0, - () => index(request, attempt + 1), - ), - ); - const ensureCommit = ( - request: Omit & {readonly commit: string}, - ) => - Effect.scoped( - Effect.gen(function* () { - const initialIdentity = yield* resolveRepositoryIdentity(request.cwd); - if ( - request.expectedIdentity && - !repositoryIdentityMatchesExpectation(initialIdentity, request.expectedIdentity) - ) { - return yield* Effect.fail(new Error('Repository identity does not match the requested graph target.')); - } - const layout = codeGraphLayout( - path, - request.threadnoteHome, - initialIdentity.checkoutId, - initialIdentity.worktreeId, - ); - const reporter = yield* withCodeGraphMaintenanceRegistration( - request.threadnoteHome, - Effect.gen(function* () { - if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph repository root is a symbolic link.')); - } - yield* fs.makeDirectory(layout.repositoryRoot, {recursive: true, mode: 0o700}); - return yield* makeCodeGraphBuildReporter({...initialIdentity, headCommit: request.commit}, layout); - }), - ); - yield* Effect.forkScoped(reporter.heartbeat); - const options = { - ...request, - onProgress: (progress: CodeGraphProgress) => - reporter.progress(progress).pipe(Effect.andThen(request.onProgress?.(progress) ?? Effect.void)), - }; - const capacityProtection: DirectPersistentCapacityProtection = { - availableDiskBytes: - options.diskCapacityAvailableBytes ?? ((target: string) => system.availableDiskBytes(target)), - crypto, - maintenance, - path, - system, - temporaryDirectory: system.tempDirectory, - walAutoCheckpointPages: options.sqliteWriterTuning?.walAutoCheckpointPages ?? 1_000, - }; - const lease = yield* withCodeGraphProcessLock( - fs, - layout.lockPath, - () => - (options.onProgress?.({phase: 'waiting', reason: 'repository-lock'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - 'ensure-commit', - Effect.gen(function* () { - if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { - return yield* Effect.fail(new Error('Code graph repository root is a symbolic link.')); - } - if (!(yield* fs.exists(layout.repositoryRoot))) { - return yield* Effect.fail(new RepositoryRegistrationLost()); - } - if (yield* codeGraphMaintenanceIntentActive(options.threadnoteHome)) { - return yield* Effect.fail(new RepositoryMaintenanceInterrupted()); - } - return yield* store - .withSession( - layout.databasePath, - Effect.gen(function* () { - const {identity: currentIdentity} = yield* resolveAndRecordCodeGraphLocalAssociation( - options.threadnoteHome, - options.cwd, - { - validateIdentity: identity => { - if (!repositoryIdentityMatchesExpectation(identity, initialIdentity)) { - return Effect.fail( - new Error('Repository identity changed while waiting for the graph lock.'), - ); - } - if ( - options.expectedIdentity && - !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity) - ) { - return Effect.fail( - new Error('Repository identity does not match the requested graph target.'), - ); - } - return Effect.void; - }, - }, - ); - yield* store.initialize(layout.databasePath); - const identity = {...currentIdentity, headCommit: options.commit}; - const cachedCommittedFileKeys = yield* cachedFileKeys(store, layout.databasePath, languagePacks); - const cacheCoalescer = cacheContentBatch({ - databasePath: layout.databasePath, - languagePacks, - onProgress: options.onProgress, - parserPool, - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ - capacityProtection, - fs, - identity, - layout, - onProgress: options.onProgress, - threadnoteHome: options.threadnoteHome, - }), - store, - threadnoteHome: options.threadnoteHome, - treeSitter, - }); - const inventory = yield* inventoryRepository(identity, { - ...options, - cachedCommittedFileKeys, - includeOverlay: false, - languagePacks, - onContentBatch: cacheCoalescer.onContentBatch, - }).pipe( - Effect.tap(() => cacheCoalescer.flush()), - Effect.ensuring(cacheCoalescer.discard().pipe(Effect.andThen(parserPool.trimIdle()))), - ); - const committedBase = yield* ensureCommittedBase({ - buildOwner: reporter.ownerIdentity, - capacityProtection, - embedding, - force: false, - fs, - identity, - inventory, - languagePacks, - layout, - onProgress: options.onProgress, - persistentMaterializationTransactionBatchLimit: - options.persistentMaterializationTransactionBatchLimit, - startedAt: yield* Clock.currentTimeMillis, - store, - threadnoteHome: options.threadnoteHome, - }); - const snapshot = committedBase.snapshot; - const leaseToken = yield* store.acquireSnapshotLease( - layout.databasePath, - snapshot.id, - 2 * 60_000, - ); - return {leaseToken, snapshot} satisfies CodeGraphCommitLease; - }), - writerSessionOptions(layout, options), - ) - .pipe( - Effect.tap(lease => reporter.completeSnapshot(lease.snapshot)), - Effect.tapError(cause => reporter.fail(cause)), - ); - }), - ).pipe( - Effect.ensuring( - runCodeGraphLifecycleOpportunity({ - maintenance, - opportunity: 'index-completion', - targets: [ - {anchorIdentity: initialIdentity, checkoutId: layout.checkoutId, databasePath: layout.databasePath}, - ], - threadnoteHome: request.threadnoteHome, - }).pipe(Effect.ignore), - ), - ); - return lease; - }), - ).pipe( - Effect.provideService(CommandExecutor, command), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(SystemInfo, system), - ); - return CodeGraphIndexer.of({ - ensureCommit, - index: options => index(options), - }); - }), - ); -} - -function withCodeGraphProcessLock( - fs: FileSystem.FileSystem, - lockPath: string, - onContention: () => Effect.Effect, - builderOperation: string, - effect: Effect.Effect, -) { - return withThreadnoteProcessActivity( - 'graph-waiter', - 'repository-lock', - withExclusiveFileLock( - fs, - lockPath, - {...CODE_GRAPH_LOCK_OPTIONS, onContention}, - withThreadnoteProcessActivity('graph-builder', builderOperation, effect), - ), - ); -} - -function writerSessionOptions(layout: CodeGraphLayout, options: CodeGraphIndexOptions) { - return { - cleanupCompletedBuildRows: true, - ...(options.onSqliteWriterConfigured ? {onSqliteWriterConfigured: options.onSqliteWriterConfigured} : {}), - onWriterContention: () => - (options.onProgress?.({phase: 'waiting', reason: 'database-writer'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - ...(options.sqliteWriterTuning ? {sqliteWriterTuning: options.sqliteWriterTuning} : {}), - writerLockPath: layout.databaseWriteLockPath, - } as const; -} - -function retiredSnapshotCleanupReporter(onProgress: CodeGraphIndexOptions['onProgress']) { - return (progress: CodeGraphRetiredSnapshotCleanupProgress) => - ( - onProgress?.({ - completed: progress.snapshotsCompleted, - pagesCompleted: progress.pagesCompleted, - phase: 'reclaiming', - rowsDeleted: progress.rowsDeleted, - total: progress.snapshotsTotal, - unit: 'snapshots', - }) ?? Effect.void - ).pipe(Effect.catch(() => Effect.void)); -} - -function withSharedCleanRequestGate(input: { - readonly checkoutId: string; - readonly effect: Effect.Effect; - readonly fs: FileSystem.FileSystem; - readonly onProgress: CodeGraphIndexOptions['onProgress']; - readonly path: Path.Path; - readonly requestedOverlay: {readonly dirty: boolean; readonly fingerprint?: string} | undefined; - readonly requestKey: string | undefined; - readonly threadnoteHome: string; -}) { - if (!input.requestKey || input.requestedOverlay?.dirty !== false) return input.effect; - return withExclusiveFileLock( - input.fs, - codeGraphRequestBuildLockPath(input.path, input.threadnoteHome, input.checkoutId, input.requestKey), - { - ...CODE_GRAPH_LOCK_OPTIONS, - onContention: () => - (input.onProgress?.({phase: 'waiting', reason: 'request-lock'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - }, - input.effect, - ); -} - -const completedConcurrentSnapshot = Effect.fn('codeGraph.completedConcurrentSnapshot')(function* ( - store: CodeGraphStoreShape, - layout: CodeGraphLayout, - identity: RepositoryIdentity, - overlay: {readonly dirty: boolean; readonly fingerprint?: string}, - requestKey: string, - requireDirectFull: boolean, -) { - const statuses = yield* readCodeGraphBuildStatuses(layout); - const completed = statuses.find( - status => status.state === 'completed' && status.request?.key === requestKey && status.result?.snapshotId, - ); - if (!completed?.result?.snapshotId) return undefined; - const ready = yield* store.currentLexicalReadySnapshotById(layout.databasePath, completed.result.snapshotId); - if ( - !ready || - ready.commit !== identity.headCommit || - ready.dirty !== overlay.dirty || - (overlay.dirty && requireDirectFull && (ready.baseSnapshotId !== undefined || !ready.id.endsWith('-direct'))) - ) { - return undefined; - } - return ready; -}); - -const prepareReadyAnalysisSummary = Effect.fn('codeGraph.prepareReadyAnalysisSummary')(function* (input: { - readonly databasePath: string; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly snapshotId: string; - readonly store: CodeGraphStoreShape; -}) { - yield* input.onProgress?.({ - phase: 'activating', - snapshotId: input.snapshotId, - subphase: 'summarizing-analysis', - }) ?? Effect.void; - return yield* ( - typeof input.store.ensureAnalysisSummary === 'function' - ? input.store.ensureAnalysisSummary(input.databasePath, input.snapshotId) - : Effect.succeed(false) - ).pipe( - Effect.ensuring( - ( - input.onProgress?.({phase: 'activating', snapshotId: input.snapshotId, subphase: 'complete'}) ?? Effect.void - ).pipe(Effect.catch(() => Effect.void)), - ), - ); -}); - -const reuseReadySnapshot = Effect.fn('codeGraph.reuseReadySnapshot')(function* (input: { - readonly embedding: CodeGraphEmbeddingIndexShape; - readonly ensureVectors: boolean; - readonly identity: RepositoryIdentity; - readonly layout: CodeGraphLayout; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly reusedFiles: number; - readonly skippedFiles: number; - readonly snapshot: CodeGraphSnapshot; - readonly startedAt: number; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; - readonly totalFiles: number; -}) { - yield* input.onProgress?.({phase: 'activating', snapshotId: input.snapshot.id, subphase: 'structural-ready'}) ?? - Effect.void; - let analysisSummaryFailure: string | undefined; - const analysisSummaryBackfilled = input.snapshot.dirty - ? yield* ( - input.onProgress?.({phase: 'activating', snapshotId: input.snapshot.id, subphase: 'complete'}) ?? Effect.void - ).pipe(Effect.as(false)) - : yield* prepareReadyAnalysisSummary({ - databasePath: input.layout.databasePath, - onProgress: input.onProgress, - snapshotId: input.snapshot.id, - store: input.store, - }).pipe( - Effect.catch(cause => - Effect.sync(() => { - analysisSummaryFailure = messageOf(cause); - return false; - }), - ), - ); - const diagnostics: string[] = analysisSummaryBackfilled - ? ['Built the persisted whole-graph analysis summary for this reused snapshot.'] - : analysisSummaryFailure - ? [`Whole-graph analysis summary will be retried lazily: ${analysisSummaryFailure}`] - : []; - if (!input.ensureVectors) { - const vectorCheck = yield* input.embedding - .check(input.threadnoteHome, input.layout, input.snapshot.id) - .pipe(Effect.catch(cause => Effect.succeed({reason: messageOf(cause), state: 'unavailable'} as const))); - if (vectorCheck.state !== 'ready') { - diagnostics.push( - `Vector graph retrieval unavailable: ${vectorCheck.reason ?? 'deferred until an explicit vector refresh'}`, - ); - } - return { - diagnostics, - durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, - identity: input.identity, - materialization: { - mode: 'reused-snapshot', - stagedFiles: 0, - totalFiles: input.totalFiles, - }, - reusedFiles: input.reusedFiles, - skippedFiles: input.skippedFiles, - snapshot: input.snapshot, - } satisfies CodeGraphIndexSummary; - } - const vectorCheck = yield* input.embedding - .check(input.threadnoteHome, input.layout, input.snapshot.id) - .pipe(Effect.catch(cause => Effect.succeed({reason: messageOf(cause), state: 'unavailable'} as const))); - const symbols = - vectorCheck.state === 'ready' - ? [] - : embeddingSymbolSource(input.store, input.layout.databasePath, input.snapshot.id); - const repaired = yield* input.embedding - .ensure(input.threadnoteHome, input.layout, input.snapshot, symbols, { - onProgress: input.onProgress, - }) - .pipe( - Effect.catch(cause => - Effect.succeed({ - embedded: 0, - ready: false, - reason: messageOf(cause), - reused: 0, - } satisfies CodeGraphEmbeddingStatus), - ), - ); - if (!repaired.ready) { - diagnostics.push(`Vector graph retrieval unavailable: ${repaired.reason ?? 'unknown reason'}`); - } - return { - diagnostics, - durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, - identity: input.identity, - materialization: { - mode: 'reused-snapshot', - stagedFiles: 0, - totalFiles: input.totalFiles, - }, - reusedFiles: input.reusedFiles, - skippedFiles: input.skippedFiles, - snapshot: input.snapshot, - } satisfies CodeGraphIndexSummary; -}); - -function codeGraphBuildRequestKey( - identity: Pick, - overlay: {readonly dirty: boolean; readonly fingerprint?: string}, - languagePacks: CodeGraphLanguagePackRegistryShape, - incrementalOverlay: boolean | undefined, -): string { - const parserIdentities = languagePacks.cacheIdentities.join('\n'); - const derivationIdentities = languagePacks.packs.map(packDerivationIdentity).sort(compareCodeUnits).join('\n'); - return sha256HexSync( - [ - 'code-graph-build-request-v3', - CODE_GRAPH_EXTRACTOR_SET_VERSION, - `lexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}`, - identity.repositoryId, - identity.checkoutId, - overlay.dirty ? identity.worktreeId : 'shared-commit', - identity.headCommit, - overlay.dirty ? (overlay.fingerprint ?? 'dirty-without-fingerprint') : 'clean', - overlay.dirty && incrementalOverlay === false ? 'direct-full' : 'default', - 'ignore-policy:3', - parserIdentities, - derivationIdentities, - ].join('\n'), - ); -} - -function sameOverlayState( - left: {readonly dirty: boolean; readonly fingerprint?: string}, - right: {readonly dirty: boolean; readonly fingerprint?: string}, -): boolean { - return left.dirty === right.dirty && (!left.dirty || left.fingerprint === right.fingerprint); -} - -function sameInventoryPaths( - left: readonly CodeGraphInventoryFile[], - right: readonly CodeGraphInventoryFile[], -): boolean { - return left.length === right.length && left.every((file, index) => file.path === right[index]?.path); -} - -function codeGraphInventoryFileChanged( - base: CodeGraphInventoryFile | undefined, - current: CodeGraphInventoryFile, - languagePacks: CodeGraphLanguagePackRegistryShape, - changedPackIds: ReadonlySet, -): boolean { - return ( - !base || - base.contentHash !== current.contentHash || - base.language !== current.language || - base.mode !== current.mode || - base.size !== current.size || - base.source !== current.source || - Option.match(languagePacks.match(current.path), { - onNone: () => false, - onSome: match => changedPackIds.has(match.pack.id), - }) - ); -} - -function inventoryFilesForPaths( - files: readonly CodeGraphInventoryFile[], - paths: readonly string[], -): readonly CodeGraphInventoryFile[] | undefined { - const selected: CodeGraphInventoryFile[] = []; - let fileIndex = 0; - for (const path of paths) { - while (fileIndex < files.length && compareCodeUnits(files[fileIndex]!.path, path) < 0) fileIndex += 1; - const file = files[fileIndex]; - if (!file || file.path !== path) return undefined; - selected.push(file); - } - return selected; -} - -const buildOwnedCleanSnapshot = Effect.fn('codeGraph.buildOwnedCleanSnapshot')(function* (input: { - readonly buildOwner: CodeGraphBuildOwnerIdentity; - readonly capacityProtection: DirectPersistentCapacityProtection; - readonly embedding: CodeGraphEmbeddingIndexShape; - readonly ensureVectors: boolean; - readonly existing: CodeGraphSnapshot | undefined; - readonly fallbackSnapshotId: string; - readonly force: boolean; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly logicalSnapshotId: string; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; - readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; - readonly startedAt: number; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; -}) { - return yield* withExclusiveFileLock( - input.fs, - codeGraphSnapshotBuildLockPath( - yield* Path.Path, - input.threadnoteHome, - input.identity.checkoutId, - input.logicalSnapshotId, - ), - { - ...CODE_GRAPH_LOCK_OPTIONS, - onContention: () => - (input.onProgress?.({phase: 'waiting', reason: 'snapshot-build'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - }, - Effect.gen(function* () { - let cleanFallbackAssessment: IncrementalOverlayAssessment | undefined; - if (!input.force) { - const ready = yield* input.store.currentLexicalReadySnapshotById( - input.layout.databasePath, - input.logicalSnapshotId, - ); - if (ready) { - if (input.existing?.id !== ready.id) { - yield* promoteReadySnapshotWithCapacity(input, ready.id); - } - return yield* reuseReadySnapshot({ - embedding: input.embedding, - ensureVectors: input.ensureVectors, - identity: input.identity, - layout: input.layout, - onProgress: input.onProgress, - reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, - skippedFiles: input.inventory.skipped, - snapshot: ready, - startedAt: input.startedAt, - store: input.store, - threadnoteHome: input.threadnoteHome, - totalFiles: input.inventory.files.length, - }); - } - const extractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); - const graphContentId = graphContentIdentity(extractorSet, input.inventory.files); - const commitReady = yield* reusableReadySnapshotForCleanCommit({ - databasePath: input.layout.databasePath, - extractorSet, - graphContentId, - headCommit: input.identity.headCommit, - repositoryId: input.identity.repositoryId, - store: input.store, - }); - if (commitReady) { - if (input.existing?.id !== commitReady.id) { - yield* promoteReadySnapshotWithCapacity(input, commitReady.id); - } - return yield* reuseReadySnapshot({ - embedding: input.embedding, - ensureVectors: input.ensureVectors, - identity: input.identity, - layout: input.layout, - onProgress: input.onProgress, - reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, - skippedFiles: input.inventory.skipped, - snapshot: commitReady, - startedAt: input.startedAt, - store: input.store, - threadnoteHome: input.threadnoteHome, - totalFiles: input.inventory.files.length, - }); - } - const workspace = - input.inventory.workspace ?? (yield* input.languagePacks.discoverWorkspace(input.inventory.files)); - const reused = yield* attemptReusableCleanSnapshot(input, workspace); - if (Option.isSome(reused)) { - if (reused.value.mode === 'complete') return reused.value.summary; - cleanFallbackAssessment = {mode: 'fallback', reason: reused.value.reason}; - } - } - const resumed = input.force - ? yield* input.store.resumableForcedBuild(input.layout.databasePath, input.logicalSnapshotId) - : undefined; - const building: CodeGraphSnapshot = resumed ?? { - commit: input.identity.headCommit, - dirty: false, - edgeCount: 0, - extractorSet: extractorSetIdentity(input.inventory.files, input.languagePacks), - fileCount: 0, - graphContentId: graphContentIdentity( - extractorSetIdentity(input.inventory.files, input.languagePacks), - input.inventory.files, - ), - id: input.fallbackSnapshotId, - repositoryId: input.identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: input.identity.worktreeId, - }; - const ownerToken = yield* input.store.claimPersistentBuild(input.layout.databasePath, input.identity, building, { - logicalSnapshotId: input.logicalSnapshotId, - owner: input.buildOwner, - }); - return yield* buildAndActivate({ - activatePointer: true, - building, - capacityProtection: input.capacityProtection, - embedding: input.embedding, - ensureVectors: input.ensureVectors, - existing: input.existing, - force: input.force, - fs: input.fs, - identity: input.identity, - incrementalAssessment: cleanFallbackAssessment, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - onProgress: input.onProgress, - persistentMaterializationTransactionBatchLimit: input.persistentMaterializationTransactionBatchLimit, - persistentOwnerToken: ownerToken, - requestedOverlay: input.requestedOverlay, - startedAt: input.startedAt, - store: input.store, - threadnoteHome: input.threadnoteHome, - }).pipe( - Effect.catch(cause => - isCodeGraphCapacityPause(cause) - ? Effect.fail(cause) - : input.store - .markFailed(input.layout.databasePath, building.id, messageOf(cause), ownerToken) - .pipe(Effect.andThen(Effect.fail(cause))), - ), - ); - }), - ); -}); - -const attemptReusableCleanSnapshot = Effect.fn('codeGraph.attemptReusableCleanSnapshot')(function* ( - input: { - readonly capacityProtection: DirectPersistentCapacityProtection; - readonly embedding: CodeGraphEmbeddingIndexShape; - readonly ensureVectors: boolean; - readonly existing: CodeGraphSnapshot | undefined; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly logicalSnapshotId: string; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; - readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; - readonly startedAt: number; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; - }, - workspace: CodeGraphWorkspace, -) { - if (!input.store.reusableCleanBase || !input.store.activateCleanSnapshotAlias) { - return Option.none(); - } - const extractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); - const preferredCommitGroups = yield* preferredIncrementalBaseCommitGroups( - input.identity.repoRoot, - input.identity.headCommit, - ); - const candidate = yield* input.store.reusableCleanBase( - input.layout.databasePath, - input.identity.repositoryId, - extractorSet, - workspace.fingerprint, - reusableBaseFileSetFingerprint(input.inventory.files), - graphContentIdentity(extractorSet, input.inventory.files), - preferredCommitGroups, - true, - ); - if (!candidate || candidate.snapshot.id === input.logicalSnapshotId) - return Option.none(); - const baseByPath = new Map(candidate.files.map(file => [file.path, file])); - if (input.inventory.files.some(file => file.source !== 'commit')) { - return Option.none(); - } - const lease = yield* input.store - .acquireSnapshotLease(input.layout.databasePath, candidate.snapshot.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) - .pipe(Effect.option); - if (Option.isNone(lease)) return Option.none(); - return yield* Effect.acquireUseRelease( - Effect.succeed(lease.value), - () => - Effect.gen(function* () { - const packDelta = - candidate.snapshot.extractorSet === extractorSet - ? ({changedPackIds: [], mode: 'compatible'} as const) - : assessCodeGraphLanguagePackDelta( - candidate.receipt.packProvenance, - input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), - ); - if ( - packDelta.mode === 'fallback' || - (candidate.snapshot.extractorSet !== extractorSet && - candidate.snapshot.extractorSet !== - extractorSetIdentityFromPackProvenance(candidate.receipt.packProvenance)) - ) { - return Option.some({mode: 'fallback', reason: 'extractor-context-changed'}); - } - const changedPackIds = new Set(packDelta.changedPackIds); - const modifiedFiles = input.inventory.files.filter(file => { - const base = baseByPath.get(file.path); - return ( - !base || - base.contentHash !== file.contentHash || - base.language !== file.language || - base.mode !== file.mode || - base.size !== file.size || - Option.match(input.languagePacks.match(file.path), { - onNone: () => false, - onSome: match => changedPackIds.has(match.pack.id), - }) - ); - }); - const currentPaths = new Set(input.inventory.files.map(file => file.path)); - const deletedPaths = candidate.files.filter(file => !currentPaths.has(file.path)).map(file => file.path); - if ( - modifiedFiles.length === 0 && - deletedPaths.length === 0 && - candidate.snapshot.extractorSet === extractorSet - ) { - const alias: CodeGraphSnapshot = { - baseSnapshotId: candidate.snapshot.id, - commit: input.identity.headCommit, - dirty: false, - edgeCount: candidate.snapshot.edgeCount, - extractorSet, - fileCount: candidate.snapshot.fileCount, - graphContentId: graphContentIdentity(extractorSet, input.inventory.files), - id: input.logicalSnapshotId, - repositoryId: input.identity.repositoryId, - state: 'ready', - symbolCount: candidate.snapshot.symbolCount, - worktreeId: input.identity.worktreeId, - }; - yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); - yield* input.store.activateCleanSnapshotAlias!( - input.layout.databasePath, - input.identity, - alias, - candidate.snapshot.id, - ); - yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); - yield* promoteReadySnapshotWithCapacity(input, alias.id); - yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); - return Option.some({ - mode: 'complete', - summary: yield* reuseReadySnapshot({ - embedding: input.embedding, - ensureVectors: input.ensureVectors, - identity: input.identity, - layout: input.layout, - onProgress: input.onProgress, - reusedFiles: input.inventory.files.length, - skippedFiles: input.inventory.skipped, - snapshot: alias, - startedAt: input.startedAt, - store: input.store, - threadnoteHome: input.threadnoteHome, - totalFiles: input.inventory.files.length, - }), - }); - } - const assessmentInput = { - candidate, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector(input), - store: input.store, - }; - const sameFileSet = deletedPaths.length === 0 && modifiedFiles.every(file => baseByPath.has(file.path)); - const boundedAssessment = sameFileSet - ? yield* assessReusableCleanBaseCompatibility(assessmentInput, workspace, modifiedFiles) - : ({mode: 'fallback', reason: 'file-set-changed'} as const); - if (boundedAssessment.mode === 'fallback') { - return Option.some(boundedAssessment); - } - const preassessment = boundedAssessment; - const committedBase: CommittedBaseResult = { - diagnostics: [], - leaseToken: Option.none(), - snapshot: candidate.snapshot, - stagingReusable: false, - }; - const building: CodeGraphSnapshot = { - baseSnapshotId: candidate.snapshot.id, - commit: input.identity.headCommit, - dirty: false, - edgeCount: 0, - extractorSet, - fileCount: 0, - graphContentId: graphContentIdentity(extractorSet, input.inventory.files), - id: input.logicalSnapshotId, - repositoryId: input.identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: input.identity.worktreeId, - }; - const incrementalAssessment = yield* assessIncrementalOverlay( - { - building, - committedBase, - force: false, - incrementalOverlayEnabled: true, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - store: input.store, - }, - workspace, - preassessment, - ); - if (incrementalAssessment.mode === 'fallback') { - return Option.some(incrementalAssessment); - } - yield* input.onProgress?.({ - completed: 0, - phase: 'materializing', - reused: input.inventory.files.length - incrementalAssessment.files.length, - total: incrementalAssessment.files.length, - unit: 'files', - }) ?? Effect.void; - const prepared = yield* input.store.preparePersistedIncrementalActivation( - input.layout.databasePath, - candidate.snapshot.id, - incrementalAssessment.files, - incrementalAssessment.facts, - { - deletedPaths: incrementalAssessment.deletedPaths, - resolutionClosure: incrementalAssessment.resolutionClosure, - }, - assessmentInput.persistentCapacityProtector, - ); - if (!prepared) { - return Option.some({mode: 'fallback', reason: 'staging-identity-mismatch'}); - } - yield* input.store.markBuilding(input.layout.databasePath, input.identity, building); - const summary = yield* buildAndActivate({ - activatePointer: true, - building, - capacityProtection: input.capacityProtection, - committedBase, - embedding: input.embedding, - ensureVectors: input.ensureVectors, - existing: input.existing, - force: false, - fs: input.fs, - identity: input.identity, - incrementalAssessment, - incrementalOverlayEnabled: true, - incrementalPrepared: true, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - onProgress: input.onProgress, - persistentMaterializationTransactionBatchLimit: input.persistentMaterializationTransactionBatchLimit, - requestedOverlay: input.requestedOverlay, - startedAt: input.startedAt, - store: input.store, - threadnoteHome: input.threadnoteHome, - workspace, - }).pipe( - Effect.catch(cause => - input.store - .markFailed(input.layout.databasePath, building.id, messageOf(cause)) - .pipe(Effect.andThen(Effect.fail(cause))), - ), - ); - return Option.some({mode: 'complete', summary}); - }), - token => input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), - ); -}); - -const attemptReusableDirtyBase = Effect.fn('codeGraph.attemptReusableDirtyBase')(function* ( - input: { - readonly extractorSet: string; - readonly identity: RepositoryIdentity; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; - readonly store: CodeGraphStoreShape; - }, - workspace: CodeGraphWorkspace, -) { - if (!input.store.reusableCleanBase) { - return Option.none<{ - readonly committedBase: CommittedBaseResult; - readonly preassessment: Extract; - }>(); - } - // Prefer the exact committed snapshot path below when it is itself a root - // reusable base. A clean incremental snapshot is already layered, so another - // overlay must instead reuse its root and include the cumulative changed set. - const committedExtractorSet = extractorSetIdentity(input.inventory.committedFiles, input.languagePacks); - const exactCommittedSnapshotId = snapshotIdentity( - input.identity, - false, - committedExtractorSet, - input.inventory.committedFiles, - ); - const exactCommittedSnapshot = yield* input.store.currentLexicalReadySnapshotById( - input.layout.databasePath, - exactCommittedSnapshotId, - ); - if (exactCommittedSnapshot && exactCommittedSnapshot.baseSnapshotId === undefined) { - return Option.none(); - } - const committedFileSetFingerprint = reusableBaseFileSetFingerprint(input.inventory.committedFiles); - const committedGraphContentId = graphContentIdentity(committedExtractorSet, input.inventory.committedFiles); - const commitReady = yield* input.store.readySnapshotForCommit( - input.layout.databasePath, - input.identity.repositoryId, - input.identity.headCommit, - committedExtractorSet, - ); - const commitReceipt = commitReady - ? yield* input.store.reusableBaseReceipt(input.layout.databasePath, commitReady.id) - : undefined; - let candidate: CodeGraphReusableCleanBase | undefined = - commitReady && - commitReceipt && - commitReady.graphContentId === committedGraphContentId && - commitReceipt.fileSetFingerprint === committedFileSetFingerprint && - commitReceipt.workspaceFingerprint === workspace.fingerprint - ? {files: input.inventory.committedFiles, receipt: commitReceipt, snapshot: commitReady} - : undefined; - if (!candidate) { - const preferredCommitGroups = yield* preferredIncrementalBaseCommitGroups( - input.identity.repoRoot, - input.identity.headCommit, - ); - candidate = yield* input.store.reusableCleanBase( - input.layout.databasePath, - input.identity.repositoryId, - input.extractorSet, - workspace.fingerprint, - reusableBaseFileSetFingerprint(input.inventory.files), - graphContentIdentity(input.extractorSet, input.inventory.files), - preferredCommitGroups, - true, - ); - } - if (!candidate) return Option.none(); - const lease = yield* input.store - .acquireSnapshotLease(input.layout.databasePath, candidate.snapshot.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) - .pipe(Effect.option); - if (Option.isNone(lease)) return Option.none(); - const leaseToken = yield* Effect.acquireRelease(Effect.succeed(lease.value), token => - input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), - ); - const packDelta = - candidate.snapshot.extractorSet === input.extractorSet - ? ({changedPackIds: [], mode: 'compatible'} as const) - : assessCodeGraphLanguagePackDelta( - candidate.receipt.packProvenance, - input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), - ); - if ( - packDelta.mode === 'fallback' || - (candidate.snapshot.extractorSet !== input.extractorSet && - candidate.snapshot.extractorSet !== extractorSetIdentityFromPackProvenance(candidate.receipt.packProvenance)) - ) { - return Option.none(); - } - const changedPackIds = new Set(packDelta.changedPackIds); - const alignedCommitCandidate = sameInventoryPaths(candidate.files, input.inventory.files); - const baseByPath = alignedCommitCandidate ? undefined : new Map(candidate.files.map(file => [file.path, file])); - const currentPaths = alignedCommitCandidate ? undefined : new Set(input.inventory.files.map(file => file.path)); - const modifiedFiles = input.inventory.files.filter((file, index) => { - const base = alignedCommitCandidate ? candidate.files[index] : baseByPath!.get(file.path); - return codeGraphInventoryFileChanged(base, file, input.languagePacks, changedPackIds); - }); - const deletedPaths = alignedCommitCandidate - ? [] - : candidate.files.filter(file => !currentPaths!.has(file.path)).map(file => file.path); - if (modifiedFiles.length === 0 && deletedPaths.length === 0) return Option.none(); - const assessmentInput = { - candidate, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - persistentCapacityProtector: input.persistentCapacityProtector, - store: input.store, - }; - const sameFileSet = - alignedCommitCandidate || (deletedPaths.length === 0 && modifiedFiles.every(file => baseByPath!.has(file.path))); - const boundedAssessment = sameFileSet - ? yield* assessReusableCleanBaseCompatibility(assessmentInput, workspace, modifiedFiles) - : ({mode: 'fallback', reason: 'file-set-changed'} as const); - if (boundedAssessment.mode === 'fallback') return Option.none(); - const preassessment = boundedAssessment; - return Option.some({ - committedBase: { - diagnostics: [ - `Dirty snapshot reused compatible persisted base ${candidate.snapshot.id} without first building commit ${input.identity.headCommit}.`, - ], - leaseToken: Option.some(leaseToken), - snapshot: candidate.snapshot, - stagingReusable: false, - }, - preassessment, - }); -}); - -const ensureCommittedBase = Effect.fn('codeGraph.ensureCommittedBase')(function* (input: { - readonly buildOwner: CodeGraphBuildOwnerIdentity; - readonly capacityProtection: DirectPersistentCapacityProtection; - readonly embedding: CodeGraphEmbeddingIndexShape; - readonly existing?: CodeGraphSnapshot; - readonly force: boolean; - readonly forceGeneration?: string; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; - readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; - readonly startedAt: number; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; -}) { - const cleanInventory: CodeGraphInventory = { - committedFiles: input.inventory.committedFiles, - committedParsedFiles: input.inventory.committedParsedFiles, - dirty: false, - files: input.inventory.committedFiles, - parsedFiles: input.inventory.committedParsedFiles, - skipped: input.inventory.skipped, - }; - const extractorSet = extractorSetIdentity(cleanInventory.files, input.languagePacks); - const logicalSnapshotId = snapshotIdentity(input.identity, false, extractorSet, cleanInventory.files); - const snapshotId = forcedSnapshotIdentity(logicalSnapshotId, input.forceGeneration); - const existing = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, snapshotId); - if (existing) { - const lease = yield* input.store - .acquireSnapshotLease(input.layout.databasePath, existing.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) - .pipe(Effect.option); - if (Option.isSome(lease)) { - const leaseToken = yield* Effect.acquireRelease(Effect.succeed(lease.value), token => - input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), - ); - return { - diagnostics: [], - leaseToken: Option.some(leaseToken), - snapshot: existing, - stagingReusable: false, - } satisfies CommittedBaseResult; - } - } - const summary = yield* withExclusiveFileLock( - input.fs, - codeGraphSnapshotBuildLockPath( - yield* Path.Path, - input.threadnoteHome, - input.identity.checkoutId, - logicalSnapshotId, - ), - { - ...CODE_GRAPH_LOCK_OPTIONS, - onContention: () => - (input.onProgress?.({phase: 'waiting', reason: 'snapshot-build'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - }, - Effect.gen(function* () { - if (!input.force) { - const ready = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, logicalSnapshotId); - if (ready) { - return { - diagnostics: [], - durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, - identity: input.identity, - materialization: {mode: 'reused-snapshot', stagedFiles: 0, totalFiles: cleanInventory.files.length}, - reusedFiles: cleanInventory.files.length - cleanInventory.parsedFiles, - skippedFiles: cleanInventory.skipped, - snapshot: ready, - } satisfies CodeGraphIndexSummary; - } - } - const resumed = input.force - ? yield* input.store.resumableForcedBuild(input.layout.databasePath, logicalSnapshotId) - : undefined; - const building: CodeGraphSnapshot = resumed ?? { - commit: input.identity.headCommit, - dirty: false, - edgeCount: 0, - extractorSet, - fileCount: 0, - graphContentId: graphContentIdentity(extractorSet, cleanInventory.files), - id: snapshotId, - repositoryId: input.identity.repositoryId, - state: 'building', - symbolCount: 0, - worktreeId: input.identity.worktreeId, - }; - const ownerToken = yield* input.store.claimPersistentBuild(input.layout.databasePath, input.identity, building, { - logicalSnapshotId, - owner: input.buildOwner, - }); - return yield* buildAndActivate({ - ...input, - activatePointer: false, - building, - ensureVectors: false, - existing: input.existing, - inventory: cleanInventory, - persistentOwnerToken: ownerToken, - }).pipe( - Effect.catch(cause => - isCodeGraphCapacityPause(cause) - ? Effect.fail(cause) - : input.store - .markFailed(input.layout.databasePath, building.id, messageOf(cause), ownerToken) - .pipe(Effect.andThen(Effect.fail(cause))), - ), - ); - }), - ); - return { - diagnostics: summary.diagnostics, - leaseToken: Option.none(), - snapshot: summary.snapshot, - // Clean builds now materialize directly into a durable `building` - // snapshot. Dirty overlays reuse the ready persisted base instead of a - // connection-private full staging graph. - stagingReusable: false, - } satisfies CommittedBaseResult; -}); - -export interface DirectPersistentCapacityContext { - readonly capacityProtection?: DirectPersistentCapacityProtection; - readonly claimMode?: CodeGraphDiskReservationOptions['claimMode']; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly layout: CodeGraphLayout; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly threadnoteHome: string; -} - -export function codeGraphDirectPersistentCapacityProtector( - input: DirectPersistentCapacityContext, -): CodeGraphDirectPersistentCapacityProtector { - return (boundary, transaction) => - input.capacityProtection - ? withCodeGraphDiskReservation( - { - boundary, - claimMode: input.claimMode, - ledgerLockPath: codeGraphDiskReservationLockPath(input.capacityProtection.path, input.threadnoteHome), - ledgerRoot: codeGraphDiskReservationRoot(input.capacityProtection.path, input.threadnoteHome), - maintenance: input.capacityProtection.maintenance - .tick({ - allowIndexPreparation: true, - anchorIdentity: input.identity, - automaticTail: false, - checkoutId: input.layout.checkoutId, - databasePath: input.layout.databasePath, - joinActive: false, - pressure: 'critical', - threadnoteHome: input.threadnoteHome, - writerLockPath: input.layout.databaseWriteLockPath, - }) - .pipe( - Effect.catch(error => (['busy', 'no-space'].includes(error.code) ? Effect.void : Effect.fail(error))), - ), - observe: observeDirectPersistentCapacity({ - boundary, - fs: input.fs, - identity: input.identity, - layout: input.layout, - protection: input.capacityProtection, - threadnoteHome: input.threadnoteHome, - }), - onDiagnostic: diagnostic => Effect.logWarning(diagnostic), - onWaiting: (input.onProgress?.({phase: 'waiting', reason: 'disk-capacity'}) ?? Effect.void).pipe( - Effect.catch(() => Effect.void), - ), - }, - transaction, - ).pipe( - Effect.provideService(Crypto.Crypto, input.capacityProtection.crypto), - Effect.provideService(FileSystem.FileSystem, input.fs), - Effect.provideService(Path.Path, input.capacityProtection.path), - Effect.provideService(SystemInfo, input.capacityProtection.system), - ) - : Effect.fail( - codeGraphDiskCapacityFailure( - { - calibrationIdentity: 'direct-persistent-capacity-unavailable', - reason: 'calibration-input-unknown', - state: 'unknown', - }, - boundary.operation, - ), - ); -} - -function promoteReadySnapshotWithCapacity( - input: DirectPersistentCapacityContext & {readonly store: CodeGraphStoreShape}, - snapshotId: string, -) { - return input.store.promote(input.layout.databasePath, input.identity, snapshotId, { - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector(input), - }); -} - -const buildAndActivate = Effect.fn('codeGraph.buildAndActivate')(function* (input: { - readonly activatePointer: boolean; - readonly building: CodeGraphSnapshot; - readonly capacityProtection: DirectPersistentCapacityProtection; - readonly committedBase?: CommittedBaseResult; - readonly existing?: CodeGraphSnapshot; - readonly embedding: CodeGraphEmbeddingIndexShape; - readonly ensureVectors: boolean; - readonly force: boolean; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly inventory: CodeGraphInventory; - readonly incrementalAssessment?: IncrementalOverlayAssessment; - readonly incrementalOverlayEnabled?: boolean; - readonly incrementalPrepared?: boolean; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; - readonly persistentOwnerToken?: string; - readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; - readonly startedAt: number; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; - readonly workspace?: CodeGraphWorkspace; -}) { - const workspace = - input.workspace ?? - input.inventory.workspace ?? - (yield* input.languagePacks.discoverWorkspace(input.inventory.files)); - const directPersistentMaterialization = input.persistentOwnerToken !== undefined; - const protectDirectPersistentWrite = codeGraphDirectPersistentCapacityProtector(input); - const persistentCapacityGuard = protectDirectPersistentWrite; - const extractionDiagnostics: string[] = [...workspace.diagnostics]; - let materializedFiles = 0; - let materializedShardFilesReused = 0; - const reusedFiles = input.inventory.files.length - input.inventory.parsedFiles; - const incrementalAssessment = - input.incrementalAssessment ?? - (input.inventory.dirty ? yield* assessIncrementalOverlay(input, workspace) : undefined); - let fallbackReason: CodeGraphOverlayFallbackReason | undefined = - incrementalAssessment?.mode === 'fallback' - ? incrementalAssessment.reason - : input.existing !== undefined && - input.existing.extractorSet !== input.building.extractorSet && - incrementalAssessment?.mode !== 'eligible' - ? 'extractor-context-changed' - : undefined; - let incrementalApplied = false; - if (incrementalAssessment?.mode === 'eligible') { - const incrementalReusedFiles = input.inventory.files.length - incrementalAssessment.files.length; - if (input.incrementalPrepared !== true) { - yield* input.onProgress?.({ - completed: 0, - phase: 'materializing', - reused: incrementalReusedFiles, - total: incrementalAssessment.files.length, - unit: 'files', - }) ?? Effect.void; - } - incrementalApplied = - input.incrementalPrepared === true - ? true - : incrementalAssessment.reuse === 'persisted-base' - ? yield* input.store.preparePersistedIncrementalActivation( - input.layout.databasePath, - input.committedBase!.snapshot.id, - incrementalAssessment.files, - incrementalAssessment.facts, - { - deletedPaths: incrementalAssessment.deletedPaths, - resolutionClosure: incrementalAssessment.resolutionClosure, - }, - protectDirectPersistentWrite, - ) - : yield* input.store.replaceStagedModifiedFiles( - input.layout.databasePath, - input.committedBase!.snapshot.id, - incrementalAssessment.files, - incrementalAssessment.facts, - protectDirectPersistentWrite, - ); - if (incrementalApplied) { - materializedFiles = incrementalAssessment.files.length; - for (const diagnostic of [ - ...input.committedBase!.diagnostics, - ...incrementalAssessment.facts.flatMap(file => file.diagnostics), - ]) { - if (extractionDiagnostics.length >= 100) break; - if (!extractionDiagnostics.includes(diagnostic)) extractionDiagnostics.push(diagnostic); - } - yield* input.onProgress?.({ - completed: materializedFiles, - phase: 'materializing', - reused: incrementalReusedFiles, - total: incrementalAssessment.files.length, - unit: 'files', - }) ?? Effect.void; - } else { - fallbackReason = 'staging-identity-mismatch'; - } - } - if (!incrementalApplied) { - const attributeFacts = createCachedCodeGraphFactsAttributor(input.inventory.files, workspace); - const shardDerivationIdentity = materializedShardDerivationIdentity( - input.building.extractorSet, - workspace.fingerprint, - graphContentIdentity(input.building.extractorSet, input.inventory.files), - ); - const sourceBytesTotal = input.inventory.files.reduce((total, file) => total + file.size, 0); - const cachedMetadata = yield* cachedFactsMetadata( - input.store, - input.layout.databasePath, - input.inventory.files, - input.languagePacks, - ); - const materializedShards = yield* input.store.loadMaterializedFileShards( - input.layout.databasePath, - input.inventory.files, - input.building.extractorSet, - shardDerivationIdentity, - ); - const materializedShardSetComplete = materializedShards.facts.size === input.inventory.files.length; - if (cachedMetadata.files !== input.inventory.files.length) { - return yield* Effect.fail( - new Error('Cached code graph facts are incomplete during materialization planning; retry with a full rebuild.'), - ); - } - const batches = factMaterializationBatches(input.inventory.files, cachedMetadata.bytesByPath); - const cachedFactBytesTotal = cachedMetadata.bytes; - const storageEstimate = estimatedMaterializationStorageBytes( - cachedFactBytesTotal, - sourceBytesTotal, - directPersistentMaterialization ? 'direct-persistent' : 'temporary-staged', - 'cached-fact-bytes', - ); - const system = yield* SystemInfo; - const [durableAvailableBytes, temporaryAvailableBytes, durableFilesystem, temporaryFilesystem] = yield* Effect.all( - [ - system.availableDiskBytes(input.layout.repositoryRoot).pipe(Effect.catch(() => Effect.succeed(undefined))), - system.availableDiskBytes(system.tempDirectory).pipe(Effect.catch(() => Effect.succeed(undefined))), - input.fs.stat(input.layout.repositoryRoot).pipe( - Effect.map(info => info.dev), - Effect.option, - ), - input.fs.stat(system.tempDirectory).pipe( - Effect.map(info => info.dev), - Effect.option, - ), - ] as const, - {concurrency: 'unbounded'}, - ); - const filesystemsShared = - Option.isSome(durableFilesystem) && Option.isSome(temporaryFilesystem) - ? durableFilesystem.value === temporaryFilesystem.value - : undefined; - const storagePlan = materializationStoragePlan(storageEstimate, { - durableAvailableBytes, - filesystemsShared, - temporaryAvailableBytes, - }); - let batchesCompleted = 0; - // Final attribution may expand one cached-fact batch into multiple bounded - // write transactions. Until each source batch is decoded, this is a lower - // bound that converges monotonically to the exact finalized receipt count. - let batchesTotal = batches.length; - let sourceBytesCompleted = 0; - let loadingMilliseconds = 0; - let attributionMilliseconds = 0; - let transactionMilliseconds = 0; - let cachedFactBytesCompleted = 0; - let factsBytesCompleted = 0; - let durableDatabaseBytes = 0; - let durableDatabaseHighWaterBytes = 0; - const storageAtStart = yield* materializationStorageFiles(input.fs, input.layout.databasePath); - let durableDatabaseFileBytes = storageAtStart.databaseBytes; - let durableDatabaseFileHighWaterBytes = storageAtStart.databaseBytes; - const durableDatabaseStartBytes = storageAtStart.databaseBytes; - let durableDatabaseGrowthBytes = 0; - let durableDatabaseGrowthHighWaterBytes = 0; - let durableFilesystemBytes = storageAtStart.totalBytes; - let durableFilesystemHighWaterBytes = storageAtStart.totalBytes; - let durableJournalBytes = storageAtStart.journalBytes; - let durableJournalHighWaterBytes = storageAtStart.journalBytes; - let durableSharedMemoryBytes = storageAtStart.sharedMemoryBytes; - let durableSharedMemoryHighWaterBytes = storageAtStart.sharedMemoryBytes; - let durableWalBytes = storageAtStart.walBytes; - let durableWalHighWaterBytes = storageAtStart.walBytes; - let lastStorageFileSampleAt = Number.NEGATIVE_INFINITY; - let temporaryDatabaseBytes = 0; - let temporaryDatabaseHighWaterBytes = 0; - let materializedRows: CodeGraphMaterializationRows = {}; - const stageMilliseconds: Partial> = {}; - const metrics = (finalFactsBytesTotal?: number): CodeGraphMaterializationMetrics => ({ - attributionMilliseconds, - batchesCompleted, - batchesTotal, - cachedFactBytesCompleted, - cachedFactBytesTotal, - ...(fallbackReason === undefined ? {} : {fallbackReason}), - factsBytesCompleted, - ...(finalFactsBytesTotal === undefined ? {} : {factsBytesTotal: finalFactsBytesTotal}), - loadingMilliseconds, - mode: 'full', - rows: materializedRows, - sourceBytesCompleted, - sourceBytesTotal, - stageMilliseconds: {...stageMilliseconds}, - storage: { - ...storagePlan, - durableDatabaseBytes, - durableDatabaseFileBytes, - durableDatabaseFileHighWaterBytes, - durableDatabaseGrowthBytes, - durableDatabaseGrowthHighWaterBytes, - durableDatabaseHighWaterBytes, - durableDatabaseStartBytes, - durableFilesystemBytes, - durableFilesystemHighWaterBytes, - durableJournalBytes, - durableJournalHighWaterBytes, - durableSharedMemoryBytes, - durableSharedMemoryHighWaterBytes, - durableWalBytes, - durableWalHighWaterBytes, - temporaryDatabaseBytes, - temporaryDatabaseHighWaterBytes, - }, - transactionMilliseconds, - }); - const refreshStorageFiles = (force = false) => - Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - if (!force && now - lastStorageFileSampleAt < 1_000) return; - const current = yield* materializationStorageFiles(input.fs, input.layout.databasePath); - durableDatabaseFileBytes = current.databaseBytes; - durableDatabaseFileHighWaterBytes = Math.max(durableDatabaseFileHighWaterBytes, current.databaseBytes); - durableDatabaseGrowthBytes = Math.max(0, current.databaseBytes - durableDatabaseStartBytes); - durableDatabaseGrowthHighWaterBytes = Math.max(durableDatabaseGrowthHighWaterBytes, durableDatabaseGrowthBytes); - durableFilesystemBytes = current.totalBytes; - durableFilesystemHighWaterBytes = Math.max(durableFilesystemHighWaterBytes, current.totalBytes); - durableJournalBytes = current.journalBytes; - durableJournalHighWaterBytes = Math.max(durableJournalHighWaterBytes, current.journalBytes); - durableSharedMemoryBytes = current.sharedMemoryBytes; - durableSharedMemoryHighWaterBytes = Math.max(durableSharedMemoryHighWaterBytes, current.sharedMemoryBytes); - durableWalBytes = current.walBytes; - durableWalHighWaterBytes = Math.max(durableWalHighWaterBytes, current.walBytes); - lastStorageFileSampleAt = now; - }); - const storageShortfalls = materializationStorageShortfalls(storagePlan); - if (storageShortfalls.length > 0) { - extractionDiagnostics.push( - `Available ${storageShortfalls.join(' and ')} disk space is below the heuristic materialization estimate; ` + - 'indexing will continue while reporting actual TEMP database usage.', - ); - } - yield* input.onProgress?.({ - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - yield* input.store.prepareActivation( - input.layout.databasePath, - input.inventory.files, - directPersistentMaterialization ? input.building.id : undefined, - undefined, - input.persistentOwnerToken, - persistentCapacityGuard, - ); - yield* input.store.stageWorkspaceCatalog(input.layout.databasePath, workspace, persistentCapacityGuard); - let persistentBatchCursor = 0; - const persistentTransactionBatchLimit = input.persistentMaterializationTransactionBatchLimit ?? 4; - interface PendingMaterializationBatch extends PersistentMaterializationTransactionCandidate { - readonly attributionMilliseconds: number; - readonly batchCachedFactBytes: number; - readonly batchFiles: readonly CodeGraphInventoryFile[]; - readonly batchIndex: number; - readonly edges: readonly CodeGraphEdge[]; - readonly loadingMilliseconds: number; - readonly monikers: readonly CodeGraphMonikerV1[]; - readonly references: readonly CodeGraphReference[]; - rows: CodeGraphMaterializationRows; - readonly stageMilliseconds: Map; - readonly symbols: readonly CodeGraphSymbol[]; - } - const pendingBatches: PendingMaterializationBatch[] = []; - const reportStagingProgress = (batch: PendingMaterializationBatch, progress: CodeGraphStagingProgress) => { - if (progress.temporaryDatabaseBytes !== undefined) { - temporaryDatabaseBytes = progress.temporaryDatabaseBytes; - temporaryDatabaseHighWaterBytes = Math.max(temporaryDatabaseHighWaterBytes, progress.temporaryDatabaseBytes); - } - if (progress.durableDatabaseBytes !== undefined) { - durableDatabaseBytes = progress.durableDatabaseBytes; - durableDatabaseHighWaterBytes = Math.max(durableDatabaseHighWaterBytes, progress.durableDatabaseBytes); - } - const activityStage = materializationStagingStage(progress); - const timingKey = progress.stage === 'committed' ? 'committing' : progress.stage; - const previousStageMilliseconds = batch.stageMilliseconds.get(timingKey) ?? 0; - const currentStageMilliseconds = progress.stageElapsedMilliseconds ?? 0; - const stageDeltaMilliseconds = Math.max(0, currentStageMilliseconds - previousStageMilliseconds); - batch.stageMilliseconds.set(timingKey, currentStageMilliseconds); - stageMilliseconds[activityStage] = (stageMilliseconds[activityStage] ?? 0) + stageDeltaMilliseconds; - batch.rows = materializationRowsWithStoreProgress(batch.rows, progress); - return refreshStorageFiles(progress.stage === 'committed').pipe( - Effect.andThen( - input.onProgress?.({ - activity: { - batchCompleted: batch.batchIndex, - batchTotal: batchesTotal, - cachedFactBytes: batch.batchCachedFactBytes, - elapsedMilliseconds: progress.elapsedMilliseconds, - factsBytes: batch.factBytes, - rows: batch.rows, - sourceBytes: batch.sourceBytes, - stage: activityStage, - stageElapsedMilliseconds: currentStageMilliseconds, - transactionMilliseconds: progress.elapsedMilliseconds, - }, - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void, - ), - Effect.catch(() => Effect.void), - ); - }; - const flushPendingBatches = () => - Effect.gen(function* () { - if (pendingBatches.length === 0) return; - const group = pendingBatches.splice(0, pendingBatches.length); - const groupByIndex = new Map(group.map(batch => [batch.batchIndex, batch])); - const transactionStartedAt = yield* Clock.currentTimeMillis; - if (directPersistentMaterialization) { - yield* input.store.stageActivationFactBatches( - input.layout.databasePath, - group.map(batch => ({ - batchIndex: batch.batchIndex, - edges: batch.edges, - finalFactBytes: batch.factBytes, - monikers: batch.monikers, - references: batch.references, - symbols: batch.symbols, - })), - (batchIndex, progress) => reportStagingProgress(groupByIndex.get(batchIndex)!, progress), - persistentCapacityGuard, - ); - } else { - for (const batch of group) { - yield* input.store.stageActivationFacts( - input.layout.databasePath, - batch.symbols, - batch.edges, - batch.references, - progress => reportStagingProgress(batch, progress), - batch.batchIndex, - persistentCapacityGuard, - batch.monikers, - ); - } - } - const groupTransactionMilliseconds = (yield* Clock.currentTimeMillis) - transactionStartedAt; - transactionMilliseconds += groupTransactionMilliseconds; - for (let index = 0; index < group.length; index += 1) { - const batch = group[index]!; - const accountedTransactionMilliseconds = index === group.length - 1 ? groupTransactionMilliseconds : 0; - materializedFiles += batch.fileCount; - batchesCompleted += 1; - sourceBytesCompleted += batch.sourceBytes; - cachedFactBytesCompleted += batch.batchCachedFactBytes; - factsBytesCompleted += batch.factBytes; - materializedRows = addMaterializationRows(materializedRows, batch.rows); - yield* input.onProgress?.({ - activity: { - batchCompleted: batch.batchIndex, - batchTotal: batchesTotal, - cachedFactBytes: batch.batchCachedFactBytes, - elapsedMilliseconds: - batch.loadingMilliseconds + batch.attributionMilliseconds + accountedTransactionMilliseconds, - factsBytes: batch.factBytes, - rows: batch.rows, - sourceBytes: batch.sourceBytes, - stage: 'committing', - transactionMilliseconds: accountedTransactionMilliseconds, - }, - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - } - }); - for (const files of batches) { - const sourceBytes = files.reduce((total, file) => total + file.size, 0); - yield* input.onProgress?.({ - activity: { - batchCompleted: batchesCompleted, - batchTotal: batchesTotal, - sourceBytes, - stage: 'loading-cache', - }, - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - const loadingStartedAt = yield* Clock.currentTimeMillis; - const missingShardFiles = materializedShardSetComplete ? [] : files; - const cached = yield* loadCachedFacts( - input.store, - input.layout.databasePath, - missingShardFiles, - input.languagePacks, - ); - const batchLoadingMilliseconds = (yield* Clock.currentTimeMillis) - loadingStartedAt; - loadingMilliseconds += batchLoadingMilliseconds; - stageMilliseconds['loading-cache'] = loadingMilliseconds; - if (missingShardFiles.some(file => !cached.facts.has(file.path))) { - return yield* Effect.fail( - new Error('A cached code graph fact disappeared during indexing; retry with a full rebuild.'), - ); - } - yield* input.onProgress?.({ - activity: { - batchCompleted: batchesCompleted, - batchTotal: batchesTotal, - cachedFactBytes: cached.bytes, - elapsedMilliseconds: batchLoadingMilliseconds, - sourceBytes, - stage: 'attributing', - }, - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - const attributionStartedAt = yield* Clock.currentTimeMillis; - const attributedMissingFacts = attributeFacts( - missingShardFiles.map(file => input.languagePacks.postprocessFile(file, cached.facts.get(file.path)!)), - ); - if (missingShardFiles.length > 0) { - yield* input.store.cacheMaterializedFileShards( - input.layout.databasePath, - missingShardFiles, - attributedMissingFacts.map(fact => serializeBoundedCodeGraphFact(fact)), - input.building.extractorSet, - shardDerivationIdentity, - protectDirectPersistentWrite, - ); - } - const attributedMissingByPath = new Map(attributedMissingFacts.map(fact => [fact.path, fact])); - const facts = files.map(file => - materializedShardSetComplete - ? materializedShards.facts.get(file.path)! - : attributedMissingByPath.get(file.path)!, - ); - materializedShardFilesReused += files.length - missingShardFiles.length; - const batchAttributionMilliseconds = (yield* Clock.currentTimeMillis) - attributionStartedAt; - attributionMilliseconds += batchAttributionMilliseconds; - stageMilliseconds.attributing = attributionMilliseconds; - const finalBatches = finalCodeGraphFactBatches(facts); - batchesTotal += Math.max(0, finalBatches.length - 1); - if (extractionDiagnostics.length < 100) { - extractionDiagnostics.push( - ...finalBatches - .flatMap(batch => batch.flatMap(value => value.facts.diagnostics)) - .slice(0, 100 - extractionDiagnostics.length), - ); - } - const filesByPath = new Map(files.map(file => [file.path, file])); - for (let finalBatchIndex = 0; finalBatchIndex < finalBatches.length; finalBatchIndex += 1) { - const finalBatch = finalBatches[finalBatchIndex]!; - const finalFacts = finalBatch.map(value => value.facts); - const batchFinalFactBytes = finalBatch.reduce((total, value) => total + value.bytes, 0); - const batchFiles = finalFacts.map(fact => filesByPath.get(fact.path)!); - const batchSourceBytes = batchFiles.reduce((total, file) => total + file.size, 0); - const batchCachedFactBytes = batchFiles.reduce( - (total, file) => total + (cachedMetadata.bytesByPath.get(file.path) ?? 0), - 0, - ); - const symbols = uniqueById(finalFacts.flatMap(file => file.symbols)); - const relationships = deduplicateMaterializationRelationships( - finalFacts.flatMap(file => file.edges), - finalFacts.flatMap(file => file.references ?? []), - ); - const edges = relationships.edges; - const references = relationships.references; - const monikers = canonicalCodeGraphMonikers(finalFacts.flatMap(file => file.monikers ?? [])); - const rows = materializationRows(symbols, edges.length, references, { - edges: relationships.duplicateEdges, - references: relationships.duplicateReferences, - }); - yield* input.onProgress?.({ - activity: { - batchCompleted: batchesCompleted, - batchTotal: batchesTotal, - cachedFactBytes: batchCachedFactBytes, - elapsedMilliseconds: finalBatchIndex === 0 ? batchAttributionMilliseconds : 0, - factsBytes: batchFinalFactBytes, - rows, - sourceBytes: batchSourceBytes, - stage: 'writing-facts', - }, - completed: materializedFiles, - metrics: metrics(), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - const candidate: PendingMaterializationBatch = { - attributionMilliseconds: finalBatchIndex === 0 ? batchAttributionMilliseconds : 0, - batchCachedFactBytes, - batchFiles, - batchIndex: persistentBatchCursor, - edges, - factBytes: batchFinalFactBytes, - fileCount: batchFiles.length, - loadingMilliseconds: finalBatchIndex === 0 ? batchLoadingMilliseconds : 0, - monikers, - references, - rows, - sourceBytes: batchSourceBytes, - stageMilliseconds: new Map(), - symbols, - }; - if ( - directPersistentMaterialization && - persistentMaterializationTransactionBatches([...pendingBatches, candidate], persistentTransactionBatchLimit) - .length > 1 - ) { - yield* flushPendingBatches(); - } - pendingBatches.push(candidate); - persistentBatchCursor += 1; - const pendingFactsBytes = pendingBatches.reduce((total, batch) => total + batch.factBytes, 0); - const pendingFiles = pendingBatches.reduce((total, batch) => total + batch.fileCount, 0); - const pendingSourceBytes = pendingBatches.reduce((total, batch) => total + batch.sourceBytes, 0); - if ( - !directPersistentMaterialization || - pendingBatches.length >= persistentTransactionBatchLimit || - pendingFiles >= PERSISTENT_MATERIALIZATION_TRANSACTION_FILES || - pendingSourceBytes >= PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES || - pendingFactsBytes >= PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES - ) { - yield* flushPendingBatches(); - } - } - } - yield* flushPendingBatches(); - batchesTotal = persistentBatchCursor; - if (directPersistentMaterialization) { - yield* input.store.finalizePersistentMaterializationPlan( - input.layout.databasePath, - persistentBatchCursor, - persistentCapacityGuard, - ); - } - yield* input.onProgress?.({ - completed: materializedFiles, - metrics: metrics(factsBytesCompleted), - phase: 'materializing', - reused: reusedFiles, - total: input.inventory.files.length, - unit: 'files', - }) ?? Effect.void; - } - const reusableBaseReceipt = input.building.dirty - ? undefined - : { - fileSetFingerprint: reusableBaseFileSetFingerprint(input.inventory.files), - packProvenance: input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), - workspaceFingerprint: workspace.fingerprint, - }; - yield* input.onProgress?.({phase: 'resolving', subphase: 'references'}) ?? Effect.void; - const resolution = yield* input.store.resolveStagedReferences( - input.layout.databasePath, - activity => - ( - input.onProgress?.({ - activity, - phase: 'resolving', - subphase: 'references', - }) ?? Effect.void - ).pipe( - Effect.catch(() => Effect.void), - Effect.andThen(Effect.yieldNow), - ), - persistentCapacityGuard, - ); - const stagedCounts = yield* input.store.stagedFactCounts(input.layout.databasePath); - yield* input.onProgress?.({ - edges: stagedCounts.edges, - phase: 'resolving', - resolved: resolution.resolved, - subphase: 'complete', - symbols: stagedCounts.symbols, - }) ?? Effect.void; - yield* input.store.shrinkMemory(input.layout.databasePath); - - const ready: CodeGraphSnapshot = { - ...input.building, - edgeCount: stagedCounts.edges, - fileCount: input.inventory.files.length, - state: 'ready', - symbolCount: stagedCounts.symbols, - }; - yield* input.onProgress?.({phase: 'activating', snapshotId: ready.id, subphase: 'validating-input'}) ?? Effect.void; - yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); - yield* input.onProgress?.({ - phase: 'activating', - snapshotId: ready.id, - subphase: 'writing-and-checkpointing', - }) ?? Effect.void; - const activatedReady = yield* Effect.gen(function* () { - const activationLease = yield* Effect.acquireRelease( - input.store.activateStaged( - input.layout.databasePath, - input.identity, - ready, - reusableBaseReceipt, - CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS, - activity => - ( - input.onProgress?.({ - activity, - phase: 'activating', - snapshotId: ready.id, - }) ?? Effect.void - ).pipe(Effect.catch(() => Effect.void)), - persistentCapacityGuard, - ), - lease => - Option.match(lease, { - onNone: () => Effect.void, - onSome: token => - input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), - }), - ); - const activated = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, ready.id); - if (!activated) { - return yield* Effect.fail(new Error('Activated code graph snapshot could not be read back from its store.')); - } - yield* input.store.shrinkMemory(input.layout.databasePath); - if (input.activatePointer) { - yield* input.onProgress?.({phase: 'activating', snapshotId: activated.id, subphase: 'promoting'}) ?? Effect.void; - // Progress callbacks are user-controlled effects and may yield long enough for - // the worktree to change. Revalidate on both sides of pointer promotion so a - // mutation observed in this window triggers the bounded retry. - yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); - yield* input.store.promote(input.layout.databasePath, input.identity, activated.id, { - persistentCapacityProtector: protectDirectPersistentWrite, - }); - yield* input.store.shrinkMemory(input.layout.databasePath); - yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); - if (Option.isSome(activationLease)) { - yield* input.store.releaseSnapshotLease(input.layout.databasePath, activationLease.value); - } - } - if (input.committedBase && Option.isSome(input.committedBase.leaseToken)) { - yield* input.store.releaseSnapshotLease(input.layout.databasePath, input.committedBase.leaseToken.value); - } - yield* input.onProgress?.({ - phase: 'activating', - snapshotId: activated.id, - subphase: input.activatePointer ? 'structural-ready' : 'complete', - }) ?? Effect.void; - return activated; - }); - let analysisSummaryFailure: string | undefined; - const analysisSummaryBackfilled = - input.activatePointer && !activatedReady.dirty - ? yield* prepareReadyAnalysisSummary({ - databasePath: input.layout.databasePath, - onProgress: input.onProgress, - snapshotId: activatedReady.id, - store: input.store, - }).pipe( - Effect.catch(cause => - Effect.sync(() => { - analysisSummaryFailure = messageOf(cause); - return false; - }), - ), - ) - : yield* ( - input.onProgress?.({ - phase: 'activating', - snapshotId: activatedReady.id, - subphase: 'complete', - }) ?? Effect.void - ).pipe(Effect.as(false)); - const embedding = input.ensureVectors - ? yield* input.embedding - .ensure( - input.threadnoteHome, - input.layout, - activatedReady, - embeddingSymbolSource(input.store, input.layout.databasePath, activatedReady.id), - { - force: input.force, - onProgress: input.onProgress, - }, - ) - .pipe( - Effect.catch(cause => - Effect.succeed({ - embedded: 0, - ready: false, - reason: messageOf(cause), - reused: 0, - } satisfies CodeGraphEmbeddingStatus), - ), - ) - : ({embedded: 0, ready: true, reused: 0} satisfies CodeGraphEmbeddingStatus); - if (input.activatePointer) { - yield* input.fs.remove(input.layout.staleMarkerPath, {force: true}).pipe(Effect.catch(() => Effect.void)); - } - return { - diagnostics: [ - ...(input.inventory.diagnostics ?? []), - ...extractionDiagnostics, - ...(input.inventory.dirty - ? [ - incrementalApplied - ? incrementalAssessment?.mode === 'eligible' && incrementalAssessment.reuse === 'persisted-base' - ? `Dirty overlay reused persisted clean base for ${materializedFiles.toLocaleString()} modified file(s).` - : `Dirty overlay reused clean staging for ${materializedFiles.toLocaleString()} modified file(s).` - : `Dirty overlay used full materialization: ${overlayFallbackDescription(fallbackReason ?? 'staging-unavailable')}.`, - ] - : incrementalApplied - ? [`Clean snapshot reused persisted base for ${materializedFiles.toLocaleString()} modified file(s).`] - : []), - ...(materializedShardFilesReused > 0 - ? [`Reused content-addressed materialized shards for ${materializedShardFilesReused.toLocaleString()} file(s).`] - : []), - ...(analysisSummaryBackfilled ? ['Built the persisted whole-graph analysis summary after promotion.'] : []), - ...(analysisSummaryFailure - ? [`Whole-graph analysis summary will be retried lazily: ${analysisSummaryFailure}`] - : []), - ...(embedding.ready ? [] : [`Vector graph retrieval unavailable: ${embedding.reason ?? 'unknown reason'}`]), - ].slice(0, 100), - durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, - identity: input.identity, - ...(incrementalApplied && incrementalAssessment?.mode === 'eligible' - ? {incrementalWork: incrementalAssessment.work} - : {}), - materialization: { - ...(incrementalApplied && incrementalAssessment?.mode === 'eligible' - ? { - ...(incrementalAssessment.closureProjects === undefined - ? {} - : {closureProjects: incrementalAssessment.closureProjects}), - ...(incrementalAssessment.resolutionClosure === undefined - ? {} - : {resolutionClosure: incrementalAssessment.resolutionClosure}), - } - : {}), - ...(fallbackReason ? {fallbackReason} : {}), - mode: incrementalApplied ? (input.inventory.dirty ? 'incremental-overlay' : 'incremental-clean') : 'full', - stagedFiles: materializedFiles, - totalFiles: input.inventory.files.length, - }, - reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, - skippedFiles: input.inventory.skipped, - snapshot: activatedReady, - } satisfies CodeGraphIndexSummary; -}); - -const assessIncrementalOverlay = Effect.fn('codeGraph.assessIncrementalOverlay')(function* ( - input: { - readonly building: CodeGraphSnapshot; - readonly committedBase?: CommittedBaseResult; - readonly force: boolean; - readonly incrementalOverlayEnabled?: boolean; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly store: CodeGraphStoreShape; - }, - workspace: CodeGraphWorkspace, - suppliedPreassessment?: IncrementalOverlayPreassessment, -) { - if (input.incrementalOverlayEnabled === false) { - return {mode: 'fallback', reason: 'disabled'} satisfies IncrementalOverlayAssessment; - } - if (input.force) return {mode: 'fallback', reason: 'forced-full-rebuild'} satisfies IncrementalOverlayAssessment; - const preassessment: IncrementalOverlayPreassessment = - suppliedPreassessment ?? - (yield* assessIncrementalOverlayCompatibility( - { - extractorSet: input.building.extractorSet, - inventory: input.inventory, - languagePacks: input.languagePacks, - layout: input.layout, - store: input.store, - }, - workspace, - )); - if (preassessment.mode === 'fallback') return preassessment; - if (!input.committedBase) - return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; - if ( - input.building.extractorSet !== input.committedBase.snapshot.extractorSet && - preassessment.extractorTransition !== true - ) { - return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayAssessment; - } - let reuse: 'persisted-base' | 'staged-base' = 'staged-base'; - if (!input.committedBase.stagingReusable) { - const receipt = yield* input.store.reusableBaseReceipt(input.layout.databasePath, input.committedBase.snapshot.id); - if ( - !receipt || - receipt.formatVersion !== CODE_GRAPH_REUSABLE_BASE_RECEIPT_VERSION || - receipt.resolutionSurfaceVersion !== 1 || - receipt.workspaceFingerprint !== preassessment.committedWorkspace.fingerprint || - (preassessment.resolutionClosure !== 'full' && - receipt.fileSetFingerprint !== reusableBaseFileSetFingerprint(input.inventory.committedFiles)) - ) { - return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; - } - reuse = 'persisted-base'; - } - let reusableFacts = preassessment.facts; - if (reuse === 'persisted-base' && preassessment.resolutionClosure !== 'full') { - const affectedPaths = - preassessment.resolutionClosure === 'project' ? new Set(preassessment.files.map(file => file.path)) : undefined; - const seeds = reusableReexportSeeds(preassessment.facts).filter(seed => !affectedPaths?.has(seed.path)); - if (seeds.length > 0) { - const reexports = yield* input.store.reusableReexports( - input.layout.databasePath, - input.committedBase.snapshot.id, - seeds, - {maxRows: 10_000}, - ); - if (reexports === undefined) { - return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; - } - if (reexports.length > 10_000) { - return {mode: 'fallback', reason: 'reexport-closure-unbounded'} satisfies IncrementalOverlayAssessment; - } - if (preassessment.resolutionClosure === 'project') { - if ( - reexports.some(reexport => affectedPaths!.has(reexport.sourcePath) || affectedPaths!.has(reexport.targetPath)) - ) { - return {mode: 'fallback', reason: 'project-closure-incomplete'} satisfies IncrementalOverlayAssessment; - } - } - const enrichedFacts = enrichPersistedTypeScriptReexports(preassessment.facts, reexports); - if (!enrichedFacts) { - return {mode: 'fallback', reason: 'reexport-closure-unbounded'} satisfies IncrementalOverlayAssessment; - } - reusableFacts = enrichedFacts; - } - } - const finalBatches = finalCodeGraphFactBatches(reusableFacts); - if (finalBatches.length !== 1 && preassessment.resolutionClosure !== 'full') { - return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayAssessment; - } - const facts = finalBatches.flatMap(batch => batch.map(value => value.facts)); - const work = measureCodeGraphIncrementalWork({ - deletedPaths: preassessment.deletedPaths, - facts, - files: preassessment.files, - totalFiles: input.inventory.files.length, - }); - if (!codeGraphIncrementalWorkFitsBudget(work)) { - return {mode: 'fallback', reason: 'incremental-rewrite-unbounded'} satisfies IncrementalOverlayAssessment; - } - return { - closureProjects: preassessment.closureProjects, - deletedPaths: preassessment.deletedPaths, - facts, - files: preassessment.files, - mode: 'eligible', - resolutionClosure: preassessment.resolutionClosure, - reuse, - work, - } satisfies IncrementalOverlayAssessment; -}); - -const assessIncrementalOverlayCompatibility = Effect.fn('codeGraph.assessIncrementalOverlayCompatibility')(function* ( - input: { - readonly extractorSet: string; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly store: CodeGraphStoreShape; - }, - workspace: CodeGraphWorkspace, -) { - if (input.extractorSet !== extractorSetIdentity(input.inventory.committedFiles, input.languagePacks)) { - return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayPreassessment; - } - const committedWorkspace = - input.inventory.workspace ?? (yield* input.languagePacks.discoverWorkspace(input.inventory.committedFiles)); - const committedByPath = new Map(input.inventory.committedFiles.map(file => [file.path, file])); - const effectiveByPath = new Map(input.inventory.files.map(file => [file.path, file])); - if ( - committedByPath.size !== effectiveByPath.size || - [...committedByPath].some(([path]) => !effectiveByPath.has(path)) - ) { - return {mode: 'fallback', reason: 'file-set-changed'} satisfies IncrementalOverlayPreassessment; - } - const modifiedFiles = input.inventory.files.filter(file => { - const committed = committedByPath.get(file.path)!; - return ( - committed.contentHash !== file.contentHash || - committed.language !== file.language || - committed.mode !== file.mode || - committed.size !== file.size || - committed.source !== file.source - ); - }); - if (modifiedFiles.length === 0) { - return {mode: 'fallback', reason: 'no-materialized-changes'} satisfies IncrementalOverlayPreassessment; - } - const workspaceCompatibility = assessCodeGraphWorkspaceCompatibility(committedWorkspace, workspace); - if (workspaceCompatibility.mode === 'fallback') { - return workspaceCompatibility satisfies IncrementalOverlayPreassessment; - } - const committedFiles = modifiedFiles.map(file => committedByPath.get(file.path)!); - const changedDecodeBudget = yield* assessProjectClosureChangedDecodeBudget({ - baseFiles: committedFiles, - currentFiles: modifiedFiles, - databasePath: input.layout.databasePath, - languagePacks: input.languagePacks, - store: input.store, - }); - if (changedDecodeBudget.mode === 'fallback') return changedDecodeBudget; - const [committedCache, effectiveCache] = yield* Effect.all( - [ - loadCachedFacts(input.store, input.layout.databasePath, committedFiles, input.languagePacks), - loadCachedFacts(input.store, input.layout.databasePath, modifiedFiles, input.languagePacks), - ], - {concurrency: 1}, - ); - if ( - committedFiles.some(file => !committedCache.facts.has(file.path)) || - modifiedFiles.some(file => !effectiveCache.facts.has(file.path)) - ) { - return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; - } - const committedRawFacts = committedFiles.map(file => - input.languagePacks.postprocessFile(file, committedCache.facts.get(file.path)!), - ); - const effectiveRawFacts = modifiedFiles.map(file => - input.languagePacks.postprocessFile(file, effectiveCache.facts.get(file.path)!), - ); - const committedFacts = attributeInventoryFacts(input.inventory.committedFiles, committedWorkspace, committedRawFacts); - const effectiveFacts = attributeInventoryFacts(input.inventory.files, workspace, effectiveRawFacts); - const committedFactsByPath = new Map(committedFacts.map(file => [file.path, file])); - const resolutionSurfaceChanged = effectiveFacts.some(file => { - const committed = committedFactsByPath.get(file.path); - return !committed || !hasSameCodeGraphResolutionSurface(committed.symbols, file.symbols); - }); - const dynamicAliases = hasDynamicAliases(committedFacts) || hasDynamicAliases(effectiveFacts); - if (!dynamicAliases && !resolutionSurfaceChanged && workspaceCompatibility.mode === 'unchanged') { - if (finalCodeGraphFactBatches(effectiveFacts).length !== 1) { - return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; - } - return { - committedWorkspace, - facts: effectiveFacts, - files: modifiedFiles, - mode: 'compatible', - } satisfies IncrementalOverlayPreassessment; - } - return yield* assessProjectIncrementalClosureCompatibility({ - baseWorkspace: committedWorkspace, - changedBaseFacts: committedFacts, - changedCurrentFacts: effectiveFacts, - currentChangedFiles: modifiedFiles, - currentFiles: input.inventory.files, - currentWorkspace: workspace, - languagePacks: input.languagePacks, - layout: input.layout, - store: input.store, - workspaceSeedProjectIds: - workspaceCompatibility.mode === 'project-closure' ? workspaceCompatibility.seedProjectIds : [], - }); -}); - -const assessReusableCleanBaseCompatibility = Effect.fn('codeGraph.assessReusableCleanBaseCompatibility')(function* ( - input: { - readonly candidate: CodeGraphReusableCleanBase; - readonly inventory: CodeGraphInventory; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; - readonly store: CodeGraphStoreShape; - }, - workspace: CodeGraphWorkspace, - modifiedFiles: readonly CodeGraphInventoryFile[], -) { - const currentExtractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); - const extractorTransition = input.candidate.snapshot.extractorSet !== currentExtractorSet; - const packDelta = extractorTransition - ? assessCodeGraphLanguagePackDelta( - input.candidate.receipt.packProvenance, - input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), - ) - : ({changedPackIds: [], mode: 'compatible'} as const); - if ( - packDelta.mode === 'fallback' || - (extractorTransition && - input.candidate.snapshot.extractorSet !== - extractorSetIdentityFromPackProvenance(input.candidate.receipt.packProvenance)) - ) { - return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayPreassessment; - } - if (input.candidate.receipt.workspaceFingerprint !== workspace.fingerprint) { - return {mode: 'fallback', reason: 'workspace-changed'} satisfies IncrementalOverlayPreassessment; - } - if (modifiedFiles.length === 0) { - return {mode: 'fallback', reason: 'no-materialized-changes'} satisfies IncrementalOverlayPreassessment; - } - const baseFiles = inventoryFilesForPaths( - input.candidate.files, - modifiedFiles.map(file => file.path), - ); - if (!baseFiles) { - return {mode: 'fallback', reason: 'file-set-changed'} satisfies IncrementalOverlayPreassessment; - } - const changedDecodeBudget = extractorTransition - ? projectClosureSourceBudgetFits(baseFiles) && projectClosureSourceBudgetFits(modifiedFiles) - ? ({mode: 'eligible'} as const) - : ({mode: 'fallback', reason: 'project-closure-unbounded'} as const) - : yield* assessProjectClosureChangedDecodeBudget({ - baseFiles, - currentFiles: modifiedFiles, - databasePath: input.layout.databasePath, - languagePacks: input.languagePacks, - store: input.store, - }); - if (changedDecodeBudget.mode === 'fallback') return changedDecodeBudget; - const [baseCache, currentCache] = yield* Effect.all( - [ - extractorTransition - ? loadCachedFactsWithPackProvenance( - input.store, - input.layout.databasePath, - baseFiles, - input.languagePacks, - input.candidate.receipt.packProvenance, - ) - : loadCachedFacts(input.store, input.layout.databasePath, baseFiles, input.languagePacks), - loadCachedFacts(input.store, input.layout.databasePath, modifiedFiles, input.languagePacks), - ], - {concurrency: 1}, - ); - if ( - baseFiles.some(file => !baseCache.facts.has(file.path)) || - modifiedFiles.some(file => !currentCache.facts.has(file.path)) - ) { - return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; - } - if ( - extractorTransition && - (baseCache.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES || - currentCache.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES) - ) { - return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; - } - const baseRawFacts = baseFiles.map(file => - input.languagePacks.postprocessFile(file, baseCache.facts.get(file.path)!), - ); - const currentRawFacts = modifiedFiles.map(file => - input.languagePacks.postprocessFile(file, currentCache.facts.get(file.path)!), - ); - const baseFacts = attributeInventoryFacts(input.candidate.files, workspace, baseRawFacts); - const currentFacts = attributeInventoryFacts(input.inventory.files, workspace, currentRawFacts); - const baseFactsByPath = new Map(baseFacts.map(file => [file.path, file])); - const resolutionSurfaceChanged = currentFacts.some(file => { - const base = baseFactsByPath.get(file.path); - return !base || !hasSameCodeGraphResolutionSurface(base.symbols, file.symbols); - }); - const dynamicAliases = hasDynamicAliases(baseFacts) || hasDynamicAliases(currentFacts); - if (dynamicAliases || resolutionSurfaceChanged) { - const closure = yield* assessProjectIncrementalClosureCompatibility({ - baseWorkspace: workspace, - changedBaseFacts: baseFacts, - changedCurrentFacts: currentFacts, - currentChangedFiles: modifiedFiles, - currentFiles: input.inventory.files, - currentWorkspace: workspace, - languagePacks: input.languagePacks, - layout: input.layout, - store: input.store, - workspaceSeedProjectIds: [], - }); - return closure.mode === 'compatible' && extractorTransition - ? {...closure, extractorTransition: true as const} - : closure; - } - if (finalCodeGraphFactBatches(currentFacts).length !== 1) { - return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; - } - yield* input.store.cacheMaterializedFileShards( - input.layout.databasePath, - modifiedFiles, - currentFacts.map(fact => serializeBoundedCodeGraphFact(fact)), - currentExtractorSet, - materializedShardDerivationIdentity( - currentExtractorSet, - workspace.fingerprint, - graphContentIdentity(currentExtractorSet, input.inventory.files), - ), - input.persistentCapacityProtector, - ); - return { - committedWorkspace: workspace, - ...(extractorTransition ? {extractorTransition: true as const} : {}), - facts: currentFacts, - files: modifiedFiles, - mode: 'compatible', - } satisfies IncrementalOverlayPreassessment; -}); - -const assessProjectIncrementalClosureCompatibility = Effect.fn( - 'codeGraph.assessProjectIncrementalClosureCompatibility', -)(function* (input: { - readonly baseWorkspace: CodeGraphWorkspace; - readonly changedBaseFacts: readonly CodeGraphFileFacts[]; - readonly changedCurrentFacts: readonly CodeGraphFileFacts[]; - readonly currentChangedFiles: readonly CodeGraphInventoryFile[]; - readonly currentFiles: readonly CodeGraphInventoryFile[]; - readonly currentWorkspace: CodeGraphWorkspace; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly layout: CodeGraphLayout; - readonly store: CodeGraphStoreShape; - readonly workspaceSeedProjectIds: readonly string[]; -}) { - const seeds = assessProjectClosureSeeds({ - committedFacts: input.changedBaseFacts, - effectiveFacts: input.changedCurrentFacts, - projects: input.currentWorkspace.projects, - }); - if (seeds.mode === 'fallback') { - return seeds satisfies IncrementalOverlayPreassessment; - } - const seedProjectIds = [...new Set([...seeds.seedProjectIds, ...input.workspaceSeedProjectIds])].sort( - compareCodeUnits, - ); - const selection = selectProjectIncrementalClosure({ - files: input.currentFiles, - modifiedPaths: input.currentChangedFiles.map(file => file.path), - projects: input.currentWorkspace.projects, - seedProjectIds, - workspaceDiagnostics: input.currentWorkspace.diagnostics, - }); - if (selection.mode === 'fallback') { - return selection satisfies IncrementalOverlayPreassessment; - } - const currentByPath = new Map(input.currentFiles.map(file => [file.path, file])); - const affectedFiles = selection.affectedPaths.map(path => currentByPath.get(path)!); - const metadata = yield* cachedFactsMetadata( - input.store, - input.layout.databasePath, - affectedFiles, - input.languagePacks, - ); - const plan = planProjectIncrementalClosure({ - cachedFactBytesByPath: metadata.bytesByPath, - files: input.currentFiles, - modifiedPaths: input.currentChangedFiles.map(file => file.path), - projects: input.currentWorkspace.projects, - seedProjectIds, - workspaceDiagnostics: input.currentWorkspace.diagnostics, - }); - if (plan.mode === 'fallback') { - return plan satisfies IncrementalOverlayPreassessment; - } - if (metadata.files !== affectedFiles.length || plan.affectedPaths.length !== affectedFiles.length) { - return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; - } - const currentCache = yield* loadCachedFacts( - input.store, - input.layout.databasePath, - affectedFiles, - input.languagePacks, - ); - if (affectedFiles.some(file => !currentCache.facts.has(file.path))) { - return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; - } - const currentRawFacts = affectedFiles.map(file => - input.languagePacks.postprocessFile(file, currentCache.facts.get(file.path)!), - ); - const currentFacts = attributeInventoryFacts(input.currentFiles, input.currentWorkspace, currentRawFacts); - if (finalCodeGraphFactBatches(currentFacts).length !== 1) { - return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; - } - return { - closureProjects: plan.projectIds.length, - committedWorkspace: input.baseWorkspace, - facts: currentFacts, - files: affectedFiles, - mode: 'compatible', - resolutionClosure: 'project', - } satisfies IncrementalOverlayPreassessment; -}); - -const assessProjectClosureChangedDecodeBudget = Effect.fn('codeGraph.assessProjectClosureChangedDecodeBudget')( - function* (input: { - readonly baseFiles: readonly CodeGraphInventoryFile[]; - readonly currentFiles: readonly CodeGraphInventoryFile[]; - readonly databasePath: string; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly store: CodeGraphStoreShape; - }) { - if (!projectClosureSourceBudgetFits(input.baseFiles) || !projectClosureSourceBudgetFits(input.currentFiles)) { - return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; - } - const [baseMetadata, currentMetadata] = yield* Effect.all( - [ - cachedFactsMetadata(input.store, input.databasePath, input.baseFiles, input.languagePacks), - cachedFactsMetadata(input.store, input.databasePath, input.currentFiles, input.languagePacks), - ], - {concurrency: 1}, - ); - if (baseMetadata.files !== input.baseFiles.length || currentMetadata.files !== input.currentFiles.length) { - return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; - } - if ( - baseMetadata.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES || - currentMetadata.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES - ) { - return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; - } - return {mode: 'eligible'} as const; - }, -); - -function projectClosureSourceBudgetFits(files: readonly CodeGraphInventoryFile[]): boolean { - if (files.length > PROJECT_INCREMENTAL_CLOSURE_MAX_FILES) return false; - let sourceBytes = 0; - for (const file of files) { - if (!Number.isSafeInteger(file.size) || file.size < 0) return false; - if (file.size > PROJECT_INCREMENTAL_CLOSURE_MAX_SOURCE_BYTES - sourceBytes) return false; - sourceBytes += file.size; - } - return true; -} - -function reusableReexportSeeds(facts: readonly CodeGraphFileFacts[]): readonly CodeGraphReusableReexportSeed[] { - const seeds = facts.flatMap(file => - (file.references ?? []).flatMap(reference => - reference.resolutionDomain === 'typescript' && isPersistedReexportEnrichableRelation(reference.relation) - ? reference.lookupTiers.flatMap(tier => tier.flatMap(parseTypeScriptPathNameLookupKey)) - : [], - ), - ); - return uniqueByKey(seeds, seed => `${seed.path}\0${seed.name}`); -} - -function enrichPersistedTypeScriptReexports( - facts: readonly CodeGraphFileFacts[], - reexports: readonly CodeGraphReusableReexport[], -): readonly CodeGraphFileFacts[] | undefined { - if (reexports.length === 0) return facts; - const provenance = new Map(); - for (const reexport of reexports) { - const key = `${reexport.sourcePath}\0${reexport.localName}`; - const values = provenance.get(key) ?? []; - values.push(reexport); - provenance.set(key, values); - } - const terminalResolver = createPersistedReexportTerminalResolver(provenance); - const enriched = facts.map(file => { - if (!file.references) return file; - return { - ...file, - references: file.references.map(reference => - enrichPersistedTypeScriptReference(reference, provenance, terminalResolver), - ), - }; - }); - return terminalResolver.exhausted() ? undefined : enriched; -} - -function enrichPersistedTypeScriptReference( - reference: CodeGraphReference, - provenance: ReadonlyMap, - terminalResolver: PersistedReexportTerminalResolver, -): CodeGraphReference { - if (reference.resolutionDomain !== 'typescript' || !isPersistedReexportEnrichableRelation(reference.relation)) { - return reference; - } - const parsedTargets = reference.lookupTiers.flatMap(tier => tier.flatMap(parseTypeScriptPathNameLookupTarget)); - if (!parsedTargets.some(target => provenance.has(`${target.path}\0${target.name}`))) return reference; - return { - ...reference, - lookupTiers: reference.lookupTiers - .map(tier => - uniqueStrings( - tier.flatMap(key => { - const parsed = parseTypeScriptPathNameLookupTarget(key); - if (parsed.length === 0) return [key]; - return parsed.flatMap(target => - (terminalResolver.resolve(target) ?? []).map( - terminal => - `${target.lookupPrefix}path:${encodeURIComponent(terminal.path)}:name:${encodeURIComponent(terminal.name)}${target.lookupSuffix}`, - ), - ); - }), - ), - ) - .filter(tier => tier.length > 0), - }; -} - -function isPersistedReexportEnrichableRelation(relation: CodeGraphEdge['relation']): boolean { - return ['calls', 'constructs', 'exports', 'extends', 'implements', 'overrides', 'references'].includes(relation); -} - -const PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS = 40_000; -const PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS = 10_000; - -type PersistedReexportTerminalTraversal = - | { - readonly mode: 'complete'; - readonly operations: number; - readonly targets: readonly CodeGraphReusableReexportSeed[]; - } - | { - readonly mode: 'fallback'; - readonly reason: 'reexport-closure-unbounded'; - }; - -interface PersistedReexportTerminalResolver { - readonly exhausted: () => boolean; - readonly resolve: (target: CodeGraphReusableReexportSeed) => readonly CodeGraphReusableReexportSeed[] | undefined; -} - -function createPersistedReexportTerminalResolver( - provenance: ReadonlyMap, -): PersistedReexportTerminalResolver { - const cache = new Map(); - let operationsRemaining = PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS; - let traversalExhausted = false; - return { - exhausted: () => traversalExhausted, - resolve: target => { - const key = reusableReexportSeedKey(target); - const cached = cache.get(key); - if (cached) return cached; - if (traversalExhausted) return undefined; - const traversal = resolvePersistedReexportTerminals(target, provenance, { - maxOperations: operationsRemaining, - maxTerminals: PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS, - }); - if (traversal.mode === 'fallback') { - traversalExhausted = true; - return undefined; - } - operationsRemaining -= traversal.operations; - cache.set(key, traversal.targets); - return traversal.targets; - }, - }; -} - -export function resolvePersistedReexportTerminals( - target: CodeGraphReusableReexportSeed, - provenance: ReadonlyMap, - options: {readonly maxOperations?: number; readonly maxTerminals?: number} = {}, -): PersistedReexportTerminalTraversal { - const maxOperations = options.maxOperations ?? PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS; - const maxTerminals = options.maxTerminals ?? PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS; - if ( - !Number.isSafeInteger(maxOperations) || - maxOperations < 0 || - !Number.isSafeInteger(maxTerminals) || - maxTerminals < 0 - ) { - return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; - } - const discovered = new Set([reusableReexportSeedKey(target)]); - const pending = [target]; - const terminals = new Map(); - let operations = 0; - const consumeOperation = (): boolean => { - if (operations >= maxOperations) return false; - operations += 1; - return true; - }; - while (pending.length > 0) { - if (!consumeOperation()) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; - const current = pending.pop()!; - const next = [...(provenance.get(reusableReexportSeedKey(current)) ?? [])].sort((left, right) => - compareCodeUnits(reusableReexportKey(left), reusableReexportKey(right)), - ); - if (next.length === 0) { - terminals.set(reusableReexportSeedKey(current), current); - if (terminals.size > maxTerminals) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; - continue; - } - for (let index = next.length - 1; index >= 0; index -= 1) { - if (!consumeOperation()) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; - const reexport = next[index]!; - const candidate = {name: reexport.importedName, path: reexport.targetPath}; - const key = reusableReexportSeedKey(candidate); - if (discovered.has(key)) continue; - discovered.add(key); - pending.push(candidate); - } - } - if (terminals.size === 0) terminals.set(reusableReexportSeedKey(target), target); - return { - mode: 'complete', - operations, - targets: [...terminals.values()].sort((left, right) => - compareCodeUnits(reusableReexportSeedKey(left), reusableReexportSeedKey(right)), - ), - }; -} - -function reusableReexportSeedKey(value: CodeGraphReusableReexportSeed): string { - return `${value.path}\0${value.name}`; -} - -function reusableReexportKey(value: CodeGraphReusableReexport): string { - return `${value.sourcePath}\0${value.localName}\0${value.targetPath}\0${value.importedName}`; -} - -function parseTypeScriptPathNameLookupKey(value: string): readonly CodeGraphReusableReexportSeed[] { - return parseTypeScriptPathNameLookupTarget(value).map(({name, path}) => ({name, path})); -} - -interface TypeScriptPathNameLookupTarget extends CodeGraphReusableReexportSeed { - readonly lookupPrefix: string; - readonly lookupSuffix: string; -} - -function parseTypeScriptPathNameLookupTarget(value: string): readonly TypeScriptPathNameLookupTarget[] { - const match = - /^typescript:((?:[^:]+:)?)path:([^:]+):name:([^:]+)(:(?:arity:\d+|implementation|merge-canonical))?$/.exec(value); - if (!match) return []; - try { - return [ - { - lookupPrefix: `typescript:${match[1]!}`, - lookupSuffix: match[4] ?? '', - name: decodeURIComponent(match[3]!), - path: decodeURIComponent(match[2]!), - }, - ]; - } catch { - return []; - } -} - -function uniqueByKey(values: readonly A[], keyOf: (value: A) => string): readonly A[] { - const output = new Map(); - for (const value of values) { - const key = keyOf(value); - if (!output.has(key)) output.set(key, value); - } - return [...output.values()]; -} - -function uniqueStrings(values: readonly string[]): readonly string[] { - return [...new Set(values)]; -} - -export function reusableBaseFileSetFingerprint(files: readonly CodeGraphInventoryFile[]): string { - return sha256HexSync( - `reusable-base-file-set-v1\n${files - .map(file => `${file.path}\0${file.language}\0${file.mode}`) - .sort(compareCodeUnits) - .join('\n')}`, - ); -} - -function attributeInventoryFacts( - files: readonly CodeGraphInventoryFile[], - workspace: CodeGraphWorkspace, - facts: readonly CodeGraphFileFacts[], -): readonly CodeGraphFileFacts[] { - return deriveCachedCodeGraphFacts(files, workspace, facts); -} - -/** - * Rehydrates parser-only cached facts into the current repository derivation. - * Resolution must precede workspace scoping because raw parser references can - * intentionally defer their lookup tiers until the whole file set is known. - */ -export function deriveCachedCodeGraphFacts( - files: readonly CodeGraphInventoryFile[], - workspace: CodeGraphWorkspace, - facts: readonly CodeGraphFileFacts[], -): readonly CodeGraphFileFacts[] { - return createCachedCodeGraphFactsAttributor(files, workspace)(facts); -} - -export function createCachedCodeGraphFactsAttributor( - files: readonly CodeGraphInventoryFile[], - workspace: CodeGraphWorkspace, -): (facts: readonly CodeGraphFileFacts[]) => readonly CodeGraphFileFacts[] { - const attributeRepositoryFacts = createRepositoryFactAttributor(files); - const attributeWorkspace = createWorkspaceAttributor(workspace); - return facts => attributeWorkspace(attributeRepositoryFacts(facts)); -} - -function hasDynamicAliases(facts: readonly CodeGraphFileFacts[]): boolean { - return facts.some(file => file.references?.some(reference => (reference.aliasLookupKeys?.length ?? 0) > 0) === true); -} - -export function hasSameCodeGraphResolutionSurface( - left: readonly CodeGraphSymbol[], - right: readonly CodeGraphSymbol[], -): boolean { - if (left.length !== right.length) return false; - const leftById = new Map(); - for (const symbol of left) { - if (leftById.has(symbol.id)) return false; - leftById.set(symbol.id, symbolResolutionSurface(symbol)); - } - const rightIds = new Set(); - for (const symbol of right) { - if (rightIds.has(symbol.id)) return false; - rightIds.add(symbol.id); - if (leftById.get(symbol.id) !== symbolResolutionSurface(symbol)) return false; - } - return true; -} - -function symbolResolutionSurface(symbol: CodeGraphSymbol): string { - // Signature, content, documentation, and spans are replaced with the changed file's facts but do not affect - // cross-file endpoint resolution. The current resolver's complete lookup contract is serialized below. - return JSON.stringify({ - arity: symbol.arity, - exported: symbol.exported, - id: symbol.id, - kind: symbol.kind, - language: symbol.language, - lookupKeys: symbol.lookupKeys ?? [], - name: symbol.name, - packageName: symbol.packageName, - path: symbol.path, - qualifiedName: symbol.qualifiedName, - resolutionDomain: symbol.resolutionDomain, - resolutionScopeId: symbol.resolutionScopeId, - }); -} - -function overlayFallbackDescription(reason: CodeGraphOverlayFallbackReason): string { - switch (reason) { - case 'cache-incomplete': - return 'cached facts were incomplete'; - case 'disabled': - return 'incremental overlay reuse was disabled'; - case 'dynamic-aliases': - return 'changed files participate in dynamic alias resolution'; - case 'extractor-context-changed': - return 'resolution context changed'; - case 'fact-budget-expanded': - return 'final attributed facts exceeded one bounded incremental transaction'; - case 'file-set-changed': - return 'eligible files were added or deleted'; - case 'forced-full-rebuild': - return 'a full rebuild was requested'; - case 'incremental-rewrite-unbounded': - return 'the changed closure exceeded the bounded incremental rewrite budget'; - case 'no-materialized-changes': - return 'no graph-eligible file content changed'; - case 'project-closure-incomplete': - return 'the declared project dependency closure was incomplete or ambiguous'; - case 'project-closure-unbounded': - return 'the project dependency closure exceeded one bounded materialization batch'; - case 'reexport-closure-unbounded': - return 'persisted reexport provenance exceeded the bounded project-closure lookup'; - case 'resolution-surface-changed': - return 'a declaration or lookup surface changed'; - case 'staging-identity-mismatch': - return 'the reusable staging identity was not current'; - case 'staging-unavailable': - return 'the compatible clean staging generation was unavailable'; - case 'workspace-changed': - return 'workspace attribution changed'; - } -} - -export interface CodeGraphCacheExtractedRow { - readonly cacheFact: BoundedCodeGraphFact; - readonly cacheIdentity: string; - readonly degraded: boolean; - readonly file: CodeGraphInventoryFile; -} - -export interface CodeGraphCacheContentCoalescer { - /** @internal Accepts already-extracted rows for bounded structural/load tests. */ - readonly acceptExtracted: ( - rows: readonly CodeGraphCacheExtractedRow[], - context: CodeGraphContentBatchContext, - ) => Effect.Effect; - /** Drops references only. This is safe in failure/cancellation cleanup because it never starts a write. */ - readonly discard: () => Effect.Effect; - /** Flushes pending rows and is called only after inventory succeeds. */ - readonly flush: () => Effect.Effect; - readonly onContentBatch: NonNullable; -} - -const CODE_GRAPH_CACHE_TIMESTAMP_CAPACITY_PLACEHOLDER = '1970-01-01T00:00:00.000Z'; - -function codeGraphFileProgressDimensions( - file: CodeGraphInventoryFile, - languagePacks: CodeGraphLanguagePackRegistryShape, -) { - const matched = Option.getOrUndefined(languagePacks.match(file.path)); - return { - classifier: matched?.pack.id ?? 'unmatched', - role: matched?.role ?? 'unmatched', - sizeBucket: codeGraphSourceSizeBucket(file.size), - } as const; -} - -/** @internal Exposed for cache coalescing/cancellation contract tests. */ -export function cacheContentBatch(options: { - readonly databasePath: string; - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; - readonly parserPool: CodeGraphParserPoolShape; - readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; - readonly store: CodeGraphStoreShape; - readonly threadnoteHome: string; - readonly treeSitter: TreeSitterRuntimeShape; -}): CodeGraphCacheContentCoalescer { - const windowSize = Math.max(1, options.parserPool.capacity * 2); - let extractionMilliseconds = 0; - let extractionFactsBytesCompleted = 0; - let extractionSourceBytesCompleted = 0; - let extractionWorkUnitsCompleted = 0; - let extractionPlan = undefined as CodeGraphContentBatchContext['extractionPlan']; - let persistenceMilliseconds = 0; - let readingMilliseconds = 0; - let pendingBytes = 0; - let pendingRows = 0; - let latestContext: CodeGraphContentBatchContext | undefined; - type PendingCacheGroup = { - readonly cacheIdentity: string; - readonly facts: BoundedCodeGraphFact[]; - readonly files: CodeGraphInventoryFile[]; - readonly paths: Set; - payloadBytes: number; - }; - const pendingGroups = new Map(); - const currentScanningMetrics = (): CodeGraphScanningMetrics | undefined => - extractionPlan === undefined - ? undefined - : { - factsBytesCompleted: extractionFactsBytesCompleted, - sourceBytesCompleted: extractionSourceBytesCompleted, - sourceBytesTotal: extractionPlan.sourceBytesTotal, - workUnitsCompleted: extractionWorkUnitsCompleted, - workUnitsTotal: extractionPlan.workUnitsTotal, - }; - const observeExtractionPlan = (plan: CodeGraphContentBatchContext['extractionPlan']) => { - if (plan === undefined) { - extractionPlan = undefined; - extractionFactsBytesCompleted = 0; - extractionSourceBytesCompleted = 0; - extractionWorkUnitsCompleted = 0; - return; - } - if ( - extractionPlan === undefined || - extractionPlan.sourceBytesTotal !== plan.sourceBytesTotal || - extractionPlan.workUnitsTotal !== plan.workUnitsTotal - ) { - extractionFactsBytesCompleted = 0; - extractionSourceBytesCompleted = 0; - extractionWorkUnitsCompleted = 0; - } - extractionPlan = plan; - }; - const completeExtractionMetrics = (file: CodeGraphInventoryFile, factsBytes: number) => { - if (extractionPlan === undefined) return undefined; - extractionFactsBytesCompleted = Math.min(Number.MAX_SAFE_INTEGER, extractionFactsBytesCompleted + factsBytes); - extractionSourceBytesCompleted = Math.min( - extractionPlan.sourceBytesTotal, - extractionSourceBytesCompleted + file.size, - ); - extractionWorkUnitsCompleted = Math.min( - extractionPlan.workUnitsTotal, - extractionWorkUnitsCompleted + - codeGraphExtractionWorkUnits(file.size, file.language, codeGraphSourceSizeBucket(file.size)), - ); - return currentScanningMetrics(); - }; - type SerializedParserResult = CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}; - const reusableExtractions = new Map(); - const reusableExtractionUses = new Map(); - const flushPendingGroup = (key: string) => - Effect.gen(function* () { - const group = pendingGroups.get(key); - if (!group || group.files.length === 0) return; - const context = latestContext; - if (!context) return yield* Effect.fail(new Error('Code graph cache persistence context is unavailable.')); - const representative = group.files[0]!; - const groupBytes = group.files.reduce((total, file) => total + file.size, 0); - const groupFactBytes = group.facts.reduce((total, fact) => total + fact.bytes, 0); - yield* emitContentProgress( - options.onProgress, - context, - { - batchCompleted: 0, - batchTotal: group.files.length, - bytes: groupBytes, - ...codeGraphFileProgressDimensions(representative, options.languagePacks), - factsBytes: groupFactBytes, - language: representative.language, - path: representative.path, - sizeBucket: codeGraphSourceSizeBucket(groupBytes), - stage: 'persisting', - }, - extractionMilliseconds, - persistenceMilliseconds, - currentScanningMetrics(), - ); - const startedAt = performance.now(); - yield* options.store.cacheFacts( - options.databasePath, - group.files, - group.facts, - group.cacheIdentity, - options.persistentCapacityProtector, - ); - const elapsed = Math.max(0, performance.now() - startedAt); - persistenceMilliseconds += elapsed; - pendingBytes -= group.payloadBytes; - pendingRows -= group.files.length; - pendingGroups.delete(key); - yield* emitContentProgress( - options.onProgress, - context, - { - batchCompleted: group.files.length, - batchTotal: group.files.length, - bytes: groupBytes, - ...codeGraphFileProgressDimensions(representative, options.languagePacks), - factsBytes: groupFactBytes, - language: representative.language, - path: representative.path, - persistMilliseconds: elapsed, - relations: group.facts.reduce((total, fact) => total + fact.facts.edges.length, 0), - sizeBucket: codeGraphSourceSizeBucket(groupBytes), - stage: 'persisting', - symbols: group.facts.reduce((total, fact) => total + fact.facts.symbols.length, 0), - }, - extractionMilliseconds, - persistenceMilliseconds, - currentScanningMetrics(), - ); - }); - const flushOldestPendingGroup = () => { - const key = pendingGroups.keys().next().value as string | undefined; - return key === undefined ? Effect.void : flushPendingGroup(key); - }; - const acceptExtracted = (rows: readonly CodeGraphCacheExtractedRow[], context: CodeGraphContentBatchContext) => - Effect.gen(function* () { - latestContext = context; - for (const {cacheFact, cacheIdentity: activeCacheIdentity, degraded, file} of rows) { - const cacheIdentity = degraded ? degradedParserCacheIdentity(activeCacheIdentity) : activeCacheIdentity; - const key = `${degraded ? 'degraded' : 'durable'}\0${cacheIdentity}`; - const reuseClass = degraded ? undefined : codeGraphBlobExtractionReuseClass(file); - const rowBytes = codeGraphFileBlobCapacityBytes({ - ...(reuseClass === undefined ? {} : {blobId: file.blobId, reuseClass}), - contentHash: file.contentHash, - createdAt: CODE_GRAPH_CACHE_TIMESTAMP_CAPACITY_PLACEHOLDER, - extractorSet: cacheIdentity, - factsJson: cacheFact.json, - path: file.path, - }); - if (rowBytes > CODE_GRAPH_CACHE_TRANSACTION_LIMITS.payloadBytes) { - return yield* Effect.fail(new Error(`Code graph cache row exceeds the persistence payload ceiling.`)); - } - while ( - pendingRows > 0 && - (pendingRows >= CODE_GRAPH_CACHE_TRANSACTION_LIMITS.rows || - pendingBytes > CODE_GRAPH_CACHE_TRANSACTION_LIMITS.payloadBytes - rowBytes) - ) { - yield* flushOldestPendingGroup(); - } - if (pendingGroups.get(key)?.paths.has(file.path)) { - // Committed-tree and dirty-overlay inventory phases can extract the - // same path with different content hashes. Both physical cache rows - // are reusable, so flush the older row instead of deduplicating it. - yield* flushPendingGroup(key); - } - const pending = pendingGroups.get(key) ?? { - cacheIdentity, - facts: [], - files: [], - paths: new Set(), - payloadBytes: 0, - }; - if (!pendingGroups.has(key)) pendingGroups.set(key, pending); - const {bytes: _bytes, content: _content, ...baseCacheFile} = file; - const cacheFile = degraded ? {...baseCacheFile, blobId: ''} : baseCacheFile; - pending.files.push(cacheFile); - pending.facts.push(cacheFact); - pending.paths.add(file.path); - pending.payloadBytes += rowBytes; - pendingBytes += rowBytes; - pendingRows += 1; - } - }); - const onContentBatch = ( - files: Parameters[0], - context: CodeGraphContentBatchContext, - ) => - Effect.gen(function* () { - readingMilliseconds += context.readingMilliseconds; - observeExtractionPlan(context.extractionPlan); - const cumulativeContext = {...context, readingMilliseconds}; - latestContext = cumulativeContext; - let parsedCompleted = 0; - const orderedFiles = [...files].sort((left, right) => compareCodeUnits(left.path, right.path)); - const localReuseCounts = new Map(); - for (const file of orderedFiles) { - const reuseKey = blobReuseKeyForFile(file, options.languagePacks); - if (reuseKey !== undefined) localReuseCounts.set(reuseKey, (localReuseCounts.get(reuseKey) ?? 0) + 1); - } - const expectedReuseCount = (key: string): number => - cumulativeContext.blobReuseCounts?.get(key) ?? localReuseCounts.get(key) ?? 0; - const finishReuseAttempt = (key: string | undefined) => { - if (key === undefined) return; - const uses = (reusableExtractionUses.get(key) ?? 0) + 1; - if (uses >= expectedReuseCount(key)) { - reusableExtractionUses.delete(key); - reusableExtractions.delete(key); - } else { - reusableExtractionUses.set(key, uses); - } - }; - for (const window of chunkValues(orderedFiles, windowSize)) { - let windowCompleted = 0; - const groups = extractionReuseGroups(window, options.languagePacks); - const extractGroup = (group: (typeof groups)[number]) => - Effect.forEach( - group.files, - file => - Effect.gen(function* () { - const reuseKey = group.reuseKey; - yield* emitContentProgress( - options.onProgress, - cumulativeContext, - { - batchCompleted: parsedCompleted, - batchTotal: files.length, - bytes: file.size, - ...codeGraphFileProgressDimensions(file, options.languagePacks), - language: file.language, - path: file.path, - stage: 'extracting', - }, - extractionMilliseconds, - persistenceMilliseconds, - currentScanningMetrics(), - ); - const donor = reuseKey === undefined ? undefined : reusableExtractions.get(reuseKey); - const reused = donor === undefined ? undefined : relocateSerializedParserResult(file, donor); - if (reused !== undefined) { - finishReuseAttempt(reuseKey); - windowCompleted += 1; - yield* emitContentProgress( - options.onProgress, - cumulativeContext, - { - batchCompleted: parsedCompleted + windowCompleted, - batchTotal: files.length, - bytes: file.size, - ...codeGraphFileProgressDimensions(file, options.languagePacks), - degraded: false, - factsBytes: reused.cacheFact.bytes, - language: file.language, - parseMilliseconds: 0, - path: file.path, - relations: reused.facts.edges.length, - stage: 'extracting', - symbols: reused.facts.symbols.length, - }, - extractionMilliseconds, - persistenceMilliseconds, - completeExtractionMetrics(file, reused.cacheFact.bytes), - ); - return {file, result: reused}; - } - const parsed = yield* extractParserFacts(file, options); - const cacheFact = serializeBoundedCodeGraphFact(parsed.facts); - const result = { - ...parsed, - cacheFact, - facts: cacheFact.facts, - } satisfies CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}; - if (!result.degraded && reuseKey !== undefined && expectedReuseCount(reuseKey) > 1) { - reusableExtractions.set(reuseKey, result); - } - finishReuseAttempt(reuseKey); - windowCompleted += 1; - yield* emitContentProgress( - options.onProgress, - cumulativeContext, - { - batchCompleted: parsedCompleted + windowCompleted, - batchTotal: files.length, - bytes: file.size, - ...codeGraphFileProgressDimensions(file, options.languagePacks), - degraded: result.degraded, - factsBytes: result.cacheFact.bytes, - language: file.language, - parseMilliseconds: result.parseMilliseconds, - path: file.path, - relations: result.facts.edges.length, - stage: 'extracting', - symbols: result.facts.symbols.length, - }, - extractionMilliseconds + result.parseMilliseconds, - persistenceMilliseconds, - completeExtractionMetrics(file, result.cacheFact.bytes), - ); - return {file, result}; - }), - {concurrency: 1}, - ); - const groupedResults: Array< - readonly {readonly file: CodeGraphInventoryFile; readonly result: SerializedParserResult}[] - > = []; - for (const lane of planCodeGraphExtractionLanes(groups, options.parserPool.capacity)) { - groupedResults.push(...(yield* Effect.forEach(lane.groups, extractGroup, {concurrency: lane.concurrency}))); - } - const results = groupedResults.flat(); - extractionMilliseconds += results.reduce((total, result) => total + result.result.parseMilliseconds, 0); - parsedCompleted += results.length; - const resultsByPath = new Map(results.map(result => [result.file.path, result.result])); - const extractedRows: CodeGraphCacheExtractedRow[] = []; - for (const group of groupFilesByCacheIdentity(window, options.languagePacks)) { - const durableFiles = group.files.filter(file => !resultsByPath.get(file.path)!.degraded); - const degradedFiles = group.files.filter(file => resultsByPath.get(file.path)!.degraded); - for (const [degraded, cacheFiles] of [ - [false, durableFiles], - [true, degradedFiles], - ] as const) { - for (const file of cacheFiles) { - extractedRows.push({ - cacheFact: resultsByPath.get(file.path)!.cacheFact, - cacheIdentity: group.cacheIdentity, - degraded, - file, - }); - } - } - } - yield* acceptExtracted(extractedRows, cumulativeContext); - } - }); - return { - acceptExtracted, - discard: () => - Effect.sync(() => { - pendingGroups.clear(); - pendingBytes = 0; - pendingRows = 0; - latestContext = undefined; - reusableExtractions.clear(); - reusableExtractionUses.clear(); - }), - flush: () => - Effect.gen(function* () { - while (pendingGroups.size > 0) yield* flushOldestPendingGroup(); - reusableExtractions.clear(); - reusableExtractionUses.clear(); - }), - onContentBatch, - }; -} - -function blobReuseKeyForFile( - file: CodeGraphInventoryFile, - languagePacks: CodeGraphLanguagePackRegistryShape, -): string | undefined { - const cacheIdentity = Option.getOrUndefined(languagePacks.cacheIdentityForPath(file.path)); - return cacheIdentity === undefined ? undefined : codeGraphBlobReuseCacheKey(file, cacheIdentity); -} - -function extractionReuseGroups( - files: readonly CodeGraphInventoryFile[], - languagePacks: CodeGraphLanguagePackRegistryShape, -): readonly {readonly files: readonly CodeGraphInventoryFile[]; readonly reuseKey?: string}[] { - const groups = new Map(); - for (const file of files) { - const reuseKey = blobReuseKeyForFile(file, languagePacks); - const key = reuseKey ?? `path\0${file.path}`; - const group = groups.get(key) ?? {files: [], ...(reuseKey === undefined ? {} : {reuseKey})}; - if (!groups.has(key)) groups.set(key, group); - group.files.push(file); - } - return [...groups.values()]; -} - -function relocateSerializedParserResult( - file: CodeGraphInventoryFile, - donor: CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}, -): (CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}) | undefined { - if (donor.degraded) return undefined; - const relocated = relocateStructuredSchemaFacts(file, donor.facts); - if (relocated === undefined) return undefined; - const cacheFact = serializeBoundedCodeGraphFact(relocated); - return {cacheFact, degraded: false, facts: cacheFact.facts, parseMilliseconds: 0}; -} - -function extractParserFacts( - file: CodeGraphInventoryFile, - options: { - readonly languagePacks: CodeGraphLanguagePackRegistryShape; - readonly parserPool: CodeGraphParserPoolShape; - readonly threadnoteHome: string; - readonly treeSitter: TreeSitterRuntimeShape; - }, -): Effect.Effect { - if (file.bytes === undefined) return options.parserPool.extract(file, options.threadnoteHome); - return Effect.gen(function* () { - const startedAt = performance.now(); - const facts = yield* options.languagePacks - .extractRawFile(file) - .pipe(Effect.provideService(TreeSitterRuntime, options.treeSitter)); - const bounded = budgetParserWorkerFacts(file, facts); - return { - degraded: bounded.degraded, - facts: bounded.facts, - parseMilliseconds: Math.max(0, performance.now() - startedAt), - }; - }); -} - -function emitContentProgress( - onProgress: ((progress: CodeGraphProgress) => Effect.Effect) | undefined, - context: CodeGraphContentBatchContext, - activity: NonNullable['activity']>, - extractionMilliseconds: number, - persistenceMilliseconds: number, - metrics?: CodeGraphScanningMetrics, -) { - return ( - onProgress?.({ - ...context.progress, - activity, - ...(metrics === undefined ? {} : {metrics}), - timings: { - extractionMilliseconds, - persistenceMilliseconds, - readingMilliseconds: context.readingMilliseconds, - }, - }) ?? Effect.void - ); -} - -function chunkValues(values: readonly A[], size: number): readonly (readonly A[])[] { - const chunks: A[][] = []; - for (let index = 0; index < values.length; index += size) chunks.push(values.slice(index, index + size)); - return chunks; -} - -function degradedParserCacheIdentity(activeIdentity: string): string { - return sha256HexSync(`code-graph-parser-degraded-v1\n${activeIdentity}`); -} - -/** Cache generations that can satisfy inventory content admission for active parser packs. */ -export function codeGraphParserCacheLookupGenerations(activeIdentities: readonly string[]): readonly { - readonly activeIdentity: string; - readonly storedIdentity: string; -}[] { - return [...new Set(activeIdentities)].sort(compareCodeUnits).flatMap(activeIdentity => [ - {activeIdentity, storedIdentity: activeIdentity}, - {activeIdentity, storedIdentity: degradedParserCacheIdentity(activeIdentity)}, - ]); -} - -/** Rebind a physical cache-generation key to the active identity expected by inventory admission. */ -export function codeGraphActiveParserCacheKey(key: string, storedIdentity: string, activeIdentity: string): string { - if (storedIdentity === activeIdentity) return key; - const terminalGeneration = `\0${storedIdentity}`; - if (key.endsWith(terminalGeneration)) return `${key.slice(0, -terminalGeneration.length)}\0${activeIdentity}`; - const embeddedGeneration = `\0${storedIdentity}\0`; - return key.includes(embeddedGeneration) ? key.replace(embeddedGeneration, () => `\0${activeIdentity}\0`) : key; -} - -const verifyIndexInput = Effect.fn('codeGraph.verifyIndexInput')(function* ( - identity: RepositoryIdentity, - verifyOverlay: boolean, - threadnoteHome: string, - requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}, -) { - const verifiedIdentity = yield* resolveRepositoryIdentity(identity.repoRoot); - if ( - !repositoryIdentityMatchesExpectation(verifiedIdentity, identity) || - (verifyOverlay && verifiedIdentity.headCommit !== identity.headCommit) - ) { - return yield* Effect.fail(new WorktreeChangedDuringIndex()); - } - if (!verifyOverlay) return; - if (!requestedOverlay) { - return yield* Effect.fail(new Error('Pointer activation requires an exact worktree build request state.')); - } - const verifiedOverlay = yield* worktreeBuildRequestState(verifiedIdentity, threadnoteHome); - if (!sameOverlayState(verifiedOverlay, requestedOverlay)) { - return yield* Effect.fail(new WorktreeChangedDuringIndex()); - } -}); - -class WorktreeChangedDuringIndex extends Error { - override readonly name = 'WorktreeChangedDuringIndex'; - - constructor() { - super('Worktree files changed during code graph indexing; retry the operation.'); - } -} - -class RepositoryRegistrationLost extends Error { - override readonly name = 'RepositoryRegistrationLost'; -} - -class RepositoryMaintenanceInterrupted extends Error { - override readonly name = 'RepositoryMaintenanceInterrupted'; - - constructor() { - super('Code graph indexing was superseded by repair or purge; retry the operation.'); - } -} - -export function extractorSetIdentity( - files: readonly {readonly contentHash: string; readonly path: string}[], - languagePacks: CodeGraphLanguagePackRegistryShape = BUILTIN_LANGUAGE_PACK_REGISTRY, -): string { - const paths = files.map(file => file.path); - return extractorSetIdentityFromIdentities( - languagePacks.activeCacheIdentities(paths), - languagePacks.activeDerivationIdentities(paths), - ); -} - -export function extractorSetIdentityFromPackProvenance(provenance: readonly CodeGraphLanguagePackProvenance[]): string { - return extractorSetIdentityFromIdentities( - [...new Set(provenance.map(pack => pack.cacheIdentity))], - [...new Set(provenance.map(pack => pack.derivationIdentity))], - ); -} - -function extractorSetIdentityFromIdentities( - cacheIdentities: readonly string[], - derivationIdentities: readonly string[], -): string { - const activeParsers = [...cacheIdentities].sort(compareCodeUnits).join('\n'); - const activeDerivations = [...derivationIdentities].sort(compareCodeUnits).join('\n'); - return sha256HexSync( - `${CODE_GRAPH_EXTRACTOR_SET_VERSION}\nactive-parser-packs:\n${activeParsers}\nactive-derivations:\n${activeDerivations}\nignore-policy:3\nresolution-context-policy:semantic-workspace-v1`, - ); -} - -export function parserCacheIdentity(): string { - const identity = BUILTIN_LANGUAGE_PACK_REGISTRY.cacheIdentityForPath('source.ts'); - return identity._tag === 'Some' ? identity.value : sha256HexSync(`${CODE_GRAPH_EXTRACTOR_SET_VERSION}:typescript`); -} - -export function snapshotIdentity( - identity: { - readonly headCommit: string; - readonly repositoryId: string; - readonly worktreeId: string; - }, - dirty: boolean, - extractorSet: string, - files: readonly {readonly contentHash: string; readonly path: string; readonly source: string}[], -): string { - const inventory = files - .map(file => `${file.path}\0${file.contentHash}\0${file.source}`) - .sort() - .join('\n'); - return `cgsn_${sha256HexSync( - `snapshot-v2\nlexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}\n${identity.repositoryId}\n${dirty ? identity.worktreeId : 'shared-commit'}\n${identity.headCommit}\n${dirty ? 'dirty' : 'clean'}\n${extractorSet}\n${inventory}`, - ).slice(0, 40)}`; -} - -/** - * Identifies the graph-producing inputs without coupling them to a Git commit or - * worktree. Commit observations remain snapshot rows and may safely alias this - * identity when the eligible inventory and derivation identity are unchanged. - */ -export function graphContentIdentity( - extractorSet: string, - files: readonly { - readonly contentHash: string; - readonly language?: string; - readonly mode?: string; - readonly path: string; - }[], -): string { - const inventory = files - .map(file => `${file.path}\0${file.contentHash}\0${file.language ?? ''}\0${file.mode ?? ''}`) - .sort() - .join('\n'); - return `cgc_${sha256HexSync( - `graph-content-v1\nlexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}\n${extractorSet}\n${inventory}`, - ).slice(0, 40)}`; -} - -export function directFullSnapshotIdentity(logicalSnapshotId: string): string { - if (!/^cgsn_[0-9a-f]{40}$/.test(logicalSnapshotId)) { - throw new Error('Logical snapshot identity is invalid.'); - } - return `${logicalSnapshotId}-direct`; -} - -function forcedSnapshotIdentity(logicalSnapshotId: string, forceGeneration: string | undefined): string { - return forceGeneration ? `${logicalSnapshotId}-full-${forceGeneration}` : logicalSnapshotId; -} - -const firstReadySnapshotById = Effect.fn('codeGraph.firstReadySnapshotById')(function* ( - store: CodeGraphStoreShape, - databasePath: string, - snapshotIds: readonly string[], -) { - for (const snapshotId of snapshotIds) { - const ready = yield* store.currentLexicalReadySnapshotById(databasePath, snapshotId); - if (ready) return ready; - } - return undefined; -}); - -/** - * Decide whether a clean ready snapshot for HEAD is graph-equivalent to the - * current inventory and safe to promote without rematerializing. - * - * Requires an explicit graphContentId on the candidate so we never promote a - * same-commit row that merely shares extractor set but not inventory content. - */ -export function shouldReuseReadySnapshotForCleanCommit(input: { - readonly candidate?: { - readonly commit: string; - readonly dirty: boolean; - readonly graphContentId?: string; - readonly id: string; - }; - readonly graphContentId: string; - readonly headCommit: string; -}): boolean { - return ( - input.candidate !== undefined && - input.candidate.dirty === false && - input.candidate.commit === input.headCommit && - input.candidate.graphContentId !== undefined && - input.candidate.graphContentId === input.graphContentId - ); -} - -const reusableReadySnapshotForCleanCommit = Effect.fn('codeGraph.reusableReadySnapshotForCleanCommit')( - function* (input: { - readonly databasePath: string; - readonly extractorSet: string; - readonly graphContentId: string; - readonly headCommit: string; - readonly repositoryId: string; - readonly store: CodeGraphStoreShape; - }) { - const candidate = yield* input.store.readySnapshotForCommit( - input.databasePath, - input.repositoryId, - input.headCommit, - input.extractorSet, - ); - return shouldReuseReadySnapshotForCleanCommit({ - candidate, - graphContentId: input.graphContentId, - headCommit: input.headCommit, - }) - ? candidate - : undefined; - }, -); - -function embeddingSymbolSource(store: CodeGraphStoreShape, databasePath: string, snapshotId: string) { - return { - count: store.countEmbeddingSymbols(databasePath, snapshotId), - loadPage: (cursor: Parameters[2], limit: number) => - store.loadEmbeddingSymbolPage(databasePath, snapshotId, cursor, limit), - }; -} - -const observeDirectPersistentCapacity = Effect.fn('codeGraph.observeDirectPersistentCapacity')(function* (input: { - readonly boundary: CodeGraphDirectPersistentCapacityBoundary; - readonly fs: FileSystem.FileSystem; - readonly identity: RepositoryIdentity; - readonly layout: CodeGraphLayout; - readonly protection: DirectPersistentCapacityProtection; - readonly threadnoteHome: string; -}) { - const [durableFilesystem, temporaryFilesystem] = yield* Effect.all( - [ - input.fs.stat(input.layout.repositoryRoot).pipe( - Effect.map(info => info.dev), - Effect.option, - ), - input.fs.stat(input.protection.temporaryDirectory).pipe( - Effect.map(info => info.dev), - Effect.option, - ), - ] as const, - {concurrency: 2}, - ); - const filesystemsShared = - Option.isSome(durableFilesystem) && Option.isSome(temporaryFilesystem) - ? durableFilesystem.value === temporaryFilesystem.value - : undefined; - const probe = (target: string) => - input.protection.availableDiskBytes(target, input.boundary).pipe(Effect.catch(() => Effect.succeed(undefined))); - const availability = - filesystemsShared === undefined - ? Effect.succeed([undefined, undefined] as const) - : filesystemsShared - ? probe(input.layout.repositoryRoot).pipe(Effect.map(available => [available, available] as const)) - : Effect.all([probe(input.layout.repositoryRoot), probe(input.protection.temporaryDirectory)] as const, { - concurrency: 2, - }); - const [[durableAvailableBytes, temporaryAvailableBytes], storage] = yield* Effect.all( - [ - availability, - inspectCodeGraphStorage(input.threadnoteHome, input.identity.checkoutId, {openWhileLocked: true}).pipe( - Effect.option, - ), - ] as const, - {concurrency: 2}, - ); - const pageStorage = - Option.isSome(storage) && storage.value.state === 'available' && storage.value.pageStorage.state === 'available' - ? storage.value.pageStorage - : undefined; - const demand = codeGraphPersistentCapacityDemand({ - boundary: input.boundary, - lexicalFormatVersion: CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION, - pageSize: pageStorage?.pageSize ?? 0, - walAutoCheckpointPages: input.protection.walAutoCheckpointPages, - }); - return { - demand, - durableAvailableBytes, - durableFilesystemKey: Option.isSome(durableFilesystem) - ? (codeGraphDiskReservationFilesystemKey(input.protection.system.platform, durableFilesystem.value) ?? - 'durable-filesystem-unknown') - : 'durable-filesystem-unknown', - freelistBytes: pageStorage?.reclaimableBytes ?? 0, - temporaryAvailableBytes, - temporaryFilesystemKey: Option.isSome(temporaryFilesystem) - ? (codeGraphDiskReservationFilesystemKey(input.protection.system.platform, temporaryFilesystem.value) ?? - 'temporary-filesystem-unknown') - : 'temporary-filesystem-unknown', - }; -}); - -function messageOf(cause: unknown): string { - return cause instanceof Error ? cause.message : String(cause); -} - -const CODE_GRAPH_LOCK_OPTIONS = { - retryIntervalMilliseconds: 100, - staleAfterMilliseconds: 120_000, - waitTimeoutMilliseconds: Number.POSITIVE_INFINITY, -} as const; - -const CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS = 10 * 60_000; -const FACT_MATERIALIZATION_BATCH_FILES = 128; -const FACT_MATERIALIZATION_BATCH_SOURCE_BYTES = 16 * 1_048_576; -const FACT_MATERIALIZATION_BATCH_CACHED_FACT_BYTES = CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM; -const PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES = 4; -const PERSISTENT_MATERIALIZATION_TRANSACTION_FILES = 512; -const PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES = 64 * 1_048_576; -const PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES = 32 * 1_048_576; -// Conservative, warning-only planning factors informed by beta.30's observed -// production-shaped live amplification. They cover indexed TEMP rows, the durable candidate -// plus WAL, rollback/subjournals, and one concurrent worktree/repository build. -// Actual high-water telemetry remains authoritative and should recalibrate -// these factors as retained release evidence grows. -const FACT_MATERIALIZATION_TEMP_FACT_AMPLIFICATION_HEURISTIC = 5; -const FACT_MATERIALIZATION_DURABLE_FACT_AMPLIFICATION_HEURISTIC = 5; -const FACT_MATERIALIZATION_JOURNAL_FACT_AMPLIFICATION_HEURISTIC = 3; -const FACT_MATERIALIZATION_TEMP_MINIMUM_ESTIMATE_BYTES = 512 * 1_048_576; -const FACT_MATERIALIZATION_DIRECT_TEMP_ESTIMATE_BYTES = 16 * 1_048_576; -const FACT_MATERIALIZATION_DURABLE_MINIMUM_ESTIMATE_BYTES = 512 * 1_048_576; -const FACT_MATERIALIZATION_JOURNAL_MINIMUM_ESTIMATE_BYTES = 256 * 1_048_576; - -export function estimatedMaterializationStorageBytes( - factBytes: number | undefined, - sourceBytes: number, - materializationMode: 'direct-persistent' | 'temporary-staged' = 'temporary-staged', - estimateBasis: 'cached-fact-bytes' | 'final-fact-bytes' = 'cached-fact-bytes', -) { - const basisBytes = factBytes ?? sourceBytes; - const estimatedTemporaryDatabaseBytes = - materializationMode === 'direct-persistent' - ? FACT_MATERIALIZATION_DIRECT_TEMP_ESTIMATE_BYTES - : Math.max( - FACT_MATERIALIZATION_TEMP_MINIMUM_ESTIMATE_BYTES, - saturatingMultiply(basisBytes, FACT_MATERIALIZATION_TEMP_FACT_AMPLIFICATION_HEURISTIC), - ); - const estimatedDurableSnapshotBytes = Math.max( - FACT_MATERIALIZATION_DURABLE_MINIMUM_ESTIMATE_BYTES, - saturatingMultiply(basisBytes, FACT_MATERIALIZATION_DURABLE_FACT_AMPLIFICATION_HEURISTIC), - ); - const estimatedJournalBytes = Math.max( - FACT_MATERIALIZATION_JOURNAL_MINIMUM_ESTIMATE_BYTES, - saturatingMultiply(basisBytes, FACT_MATERIALIZATION_JOURNAL_FACT_AMPLIFICATION_HEURISTIC), - ); - const estimatedConcurrentBuildBytes = saturatingAdd( - estimatedTemporaryDatabaseBytes, - estimatedDurableSnapshotBytes, - estimatedJournalBytes, - ); - return { - estimateBasis: factBytes === undefined ? ('source-bytes-fallback' as const) : estimateBasis, - estimatedConcurrentBuildBytes, - estimatedDurableSnapshotBytes, - estimatedJournalBytes, - estimatedRequiredBytes: saturatingAdd(estimatedConcurrentBuildBytes, estimatedConcurrentBuildBytes), - estimatedTemporaryDatabaseBytes, - materializationMode, - }; -} - -export interface MaterializationStorageAvailability { - readonly durableAvailableBytes?: number; - readonly filesystemsShared?: boolean; - readonly temporaryAvailableBytes?: number; -} - -export type MaterializationStoragePlan = ReturnType & - MaterializationStorageAvailability & { - readonly availableBytes?: number; - readonly estimatedDurableFilesystemRequiredBytes: number; - readonly estimatedTemporaryFilesystemRequiredBytes: number; - }; - -/** - * Plans warning-only materialization headroom for SQLite's durable and TEMP - * filesystems. A second complete allowance covers one concurrent worktree or - * repository build without imposing a repository-size rejection. - */ -export function materializationStoragePlan( - estimate: ReturnType, - availability: MaterializationStorageAvailability, -): MaterializationStoragePlan { - const estimatedDurableFilesystemRequiredBytes = saturatingMultiply( - estimate.materializationMode === 'direct-persistent' - ? saturatingAdd(estimate.estimatedDurableSnapshotBytes, estimate.estimatedJournalBytes) - : estimate.estimatedDurableSnapshotBytes, - 2, - ); - const estimatedTemporaryFilesystemRequiredBytes = saturatingMultiply( - estimate.materializationMode === 'direct-persistent' - ? estimate.estimatedTemporaryDatabaseBytes - : saturatingAdd(estimate.estimatedTemporaryDatabaseBytes, estimate.estimatedJournalBytes), - 2, - ); - const sharedAvailableBytes = - availability.filesystemsShared === true - ? minimumDefined(availability.durableAvailableBytes, availability.temporaryAvailableBytes) - : undefined; - return { - ...estimate, - ...availability, - ...(sharedAvailableBytes === undefined ? {} : {availableBytes: sharedAvailableBytes}), - estimatedDurableFilesystemRequiredBytes, - estimatedTemporaryFilesystemRequiredBytes, - }; -} - -export function materializationStorageShortfalls(storage: { - readonly availableBytes?: number; - readonly durableAvailableBytes?: number; - readonly estimatedDurableFilesystemRequiredBytes?: number; - readonly estimatedRequiredBytes?: number; - readonly estimatedTemporaryFilesystemRequiredBytes?: number; - readonly filesystemsShared?: boolean; - readonly temporaryAvailableBytes?: number; -}): readonly ('durable' | 'shared' | 'temporary')[] { - if (storage.filesystemsShared === true) { - return storage.availableBytes !== undefined && - storage.estimatedRequiredBytes !== undefined && - storage.availableBytes < storage.estimatedRequiredBytes - ? ['shared'] - : []; - } - const shortfalls: ('durable' | 'temporary')[] = []; - if ( - storage.durableAvailableBytes !== undefined && - storage.estimatedDurableFilesystemRequiredBytes !== undefined && - storage.durableAvailableBytes < storage.estimatedDurableFilesystemRequiredBytes - ) { - shortfalls.push('durable'); - } - if ( - storage.temporaryAvailableBytes !== undefined && - storage.estimatedTemporaryFilesystemRequiredBytes !== undefined && - storage.temporaryAvailableBytes < storage.estimatedTemporaryFilesystemRequiredBytes - ) { - shortfalls.push('temporary'); - } - return shortfalls; -} - -function minimumDefined(left: number | undefined, right: number | undefined): number | undefined { - if (left === undefined) return right; - if (right === undefined) return left; - return Math.min(left, right); -} - -function saturatingMultiply(value: number, multiplier: number): number { - return Math.min(Number.MAX_SAFE_INTEGER, value * multiplier); -} - -function saturatingAdd(...values: readonly number[]): number { - return values.reduce((total, value) => Math.min(Number.MAX_SAFE_INTEGER, total + value), 0); -} - -export function factMaterializationBatches( - values: readonly T[], - cachedFactBytesByPath: ReadonlyMap = new Map(), -): readonly (readonly T[])[] { - const output: T[][] = []; - let batch: T[] = []; - let batchBytes = 0; - let batchFactBytes = 0; - for (const value of values) { - // Current-version cache writes and materialization reads both apply the - // same per-file compactor. Clamp defensive metadata from an unexpected - // legacy/corrupt row to that in-memory materialization ceiling, so there - // is no oversized-singleton exception in the batch planner. - const factBytes = Math.min( - CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM, - Math.max(0, cachedFactBytesByPath.get(value.path) ?? 0), - ); - if ( - batch.length > 0 && - (batch.length >= FACT_MATERIALIZATION_BATCH_FILES || - batchBytes + value.size > FACT_MATERIALIZATION_BATCH_SOURCE_BYTES || - batchFactBytes + factBytes > FACT_MATERIALIZATION_BATCH_CACHED_FACT_BYTES) - ) { - output.push(batch); - batch = []; - batchBytes = 0; - batchFactBytes = 0; - } - batch.push(value); - batchBytes += value.size; - batchFactBytes += factBytes; - } - if (batch.length > 0) output.push(batch); - return output; -} - -export interface PersistentMaterializationTransactionCandidate { - readonly factBytes: number; - readonly fileCount: number; - readonly sourceBytes: number; -} - -/** - * Coalesces contiguous, already-bounded logical receipts into larger physical - * SQLite transactions. Logical receipt identities stay unchanged so an - * interrupted build from an older release resumes without replay or graph - * drift. A candidate over a physical ceiling remains an isolated singleton. - */ -export function persistentMaterializationTransactionBatches( - values: readonly T[], - maximumBatches = PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES, -): readonly (readonly T[])[] { - const batchLimit = Math.max(1, Math.min(PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES, maximumBatches)); - const output: T[][] = []; - let batch: T[] = []; - let factBytes = 0; - let fileCount = 0; - let sourceBytes = 0; - for (const value of values) { - if ( - batch.length > 0 && - (batch.length >= batchLimit || - fileCount + value.fileCount > PERSISTENT_MATERIALIZATION_TRANSACTION_FILES || - sourceBytes + value.sourceBytes > PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES || - factBytes + value.factBytes > PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES) - ) { - output.push(batch); - batch = []; - factBytes = 0; - fileCount = 0; - sourceBytes = 0; - } - batch.push(value); - factBytes += value.factBytes; - fileCount += value.fileCount; - sourceBytes += value.sourceBytes; - } - if (batch.length > 0) output.push(batch); - return output; -} - -function uniqueById(values: readonly T[]): readonly T[] { - const unique = new Map(); - for (const value of values) { - if (!unique.has(value.id)) unique.set(value.id, value); - } - return [...unique.values()]; -} - -/** - * Extraction may encounter the same relationship repeatedly at one call site - * or through overlapping language-pack derivations. The storage layer keeps - * strict INSERT semantics; collapse those logical duplicates deterministically - * before they reach its primary-key boundary. - */ -export function deduplicateMaterializationRelationships( - edges: readonly CodeGraphEdge[], - references: readonly CodeGraphReference[], -): { - readonly duplicateEdges: number; - readonly duplicateReferences: number; - readonly edges: readonly CodeGraphEdge[]; - readonly references: readonly CodeGraphReference[]; -} { - const edgeById = new Map(); - for (const edge of edges) { - if (!edgeById.has(edge.id)) edgeById.set(edge.id, edge); - } - const referenceByEdgeId = new Map(); - for (const reference of references) { - // Reference attribution has historically been last-wins for one logical - // edge. Preserve that contract for older, uncompacted cache rows while - // edges retain their first stable evidence occurrence. - referenceByEdgeId.set(reference.edgeId, reference); - } - return { - duplicateEdges: edges.length - edgeById.size, - duplicateReferences: references.length - referenceByEdgeId.size, - edges: [...edgeById.values()], - references: [...referenceByEdgeId.values()], - }; -} - -function materializationRows( - symbols: readonly CodeGraphSymbol[], - edges: number, - references: readonly CodeGraphReference[], - deduplicated: {readonly edges: number; readonly references: number}, -): CodeGraphMaterializationRows { - return { - deduplicatedEdges: deduplicated.edges, - deduplicatedReferences: deduplicated.references, - edges, - lookupKeys: symbols.reduce((total, symbol) => total + (symbol.lookupKeys?.length ?? 0), 0), - referenceCandidates: references.reduce( - (total, reference) => total + reference.lookupTiers.reduce((tierTotal, tier) => tierTotal + tier.length, 0), - 0, - ), - references: references.length, - symbols: symbols.length, - }; -} - -export function addMaterializationRows( - left: CodeGraphMaterializationRows, - right: CodeGraphMaterializationRows, -): CodeGraphMaterializationRows { - return { - deduplicatedEdges: (left.deduplicatedEdges ?? 0) + (right.deduplicatedEdges ?? 0), - deduplicatedReferences: (left.deduplicatedReferences ?? 0) + (right.deduplicatedReferences ?? 0), - edges: (left.edges ?? 0) + (right.edges ?? 0), - lookupKeys: (left.lookupKeys ?? 0) + (right.lookupKeys ?? 0), - referenceCandidates: (left.referenceCandidates ?? 0) + (right.referenceCandidates ?? 0), - references: (left.references ?? 0) + (right.references ?? 0), - reexports: (left.reexports ?? 0) + (right.reexports ?? 0), - symbols: (left.symbols ?? 0) + (right.symbols ?? 0), - terms: (left.terms ?? 0) + (right.terms ?? 0), - }; -} - -export function materializationRowsWithStoreProgress( - rows: CodeGraphMaterializationRows, - progress: CodeGraphStagingProgress, -): CodeGraphMaterializationRows { - // Store observers emit a zero-row stage boundary before the first bounded - // statement. Keep the batch estimate at that boundary; replacing it with - // zero made the CLI claim that a non-empty batch contained no symbols or - // lookup keys. Positive observations monotonically replace estimates with - // the rows actually accepted by SQLite. - if (progress.rowsCompleted === 0) return rows; - switch (progress.stage) { - case 'symbols': - return {...rows, symbols: progress.rowsCompleted}; - case 'lookup-keys': - return {...rows, lookupKeys: progress.rowsCompleted}; - case 'terms': - return {...rows, terms: progress.rowsCompleted}; - case 'edges': - return {...rows, edges: progress.rowsCompleted}; - case 'references': - return {...rows, references: progress.rowsCompleted}; - case 'reference-candidates': - return {...rows, referenceCandidates: progress.rowsCompleted}; - case 'reexports': - return {...rows, reexports: progress.rowsCompleted}; - case 'analysis': - case 'receipt': - case 'validating': - case 'committing': - case 'committed': - return rows; - } -} - -interface MaterializationStorageFiles { - readonly databaseBytes: number; - readonly journalBytes: number; - readonly sharedMemoryBytes: number; - readonly totalBytes: number; - readonly walBytes: number; -} - -function materializationStorageFiles( - fs: FileSystem.FileSystem, - databasePath: string, -): Effect.Effect { - const bytes = (file: string) => - fs.stat(file).pipe( - Effect.map(info => Math.min(Number(info.size), Number.MAX_SAFE_INTEGER)), - Effect.catch(() => Effect.succeed(0)), - ); - return Effect.all( - [bytes(databasePath), bytes(`${databasePath}-journal`), bytes(`${databasePath}-shm`), bytes(`${databasePath}-wal`)], - {concurrency: 4}, - ).pipe( - Effect.map(([databaseBytes, journalBytes, sharedMemoryBytes, walBytes]) => ({ - databaseBytes, - journalBytes, - sharedMemoryBytes, - totalBytes: databaseBytes + journalBytes + sharedMemoryBytes + walBytes, - walBytes, - })), - ); -} - -function materializationStagingStage( - progress: CodeGraphStagingProgress, -): NonNullable['activity']>['stage'] { - switch (progress.stage) { - case 'validating': - return 'preparing-rows'; - case 'symbols': - return 'writing-symbols'; - case 'lookup-keys': - return 'writing-lookups'; - case 'terms': - return 'writing-terms'; - case 'edges': - return 'writing-edges'; - case 'reference-candidates': - return 'writing-candidates'; - case 'references': - case 'reexports': - return 'writing-references'; - case 'analysis': - return 'writing-analysis'; - case 'receipt': - return 'writing-receipt'; - case 'committing': - case 'committed': - return 'committing'; - } -} - -function cachedFileKeys( - store: CodeGraphStoreShape, - databasePath: string, - languagePacks: CodeGraphLanguagePackRegistryShape, -): Effect.Effect, unknown> { - return Effect.forEach( - codeGraphParserCacheLookupGenerations(languagePacks.cacheIdentities), - generation => - store - .cachedCommittedFileKeys(databasePath, generation.storedIdentity) - .pipe( - Effect.map( - keys => - new Set( - [...keys].map(key => - codeGraphActiveParserCacheKey(key, generation.storedIdentity, generation.activeIdentity), - ), - ), - ), - ), - {concurrency: 1}, - ).pipe(Effect.map(sets => new Set(sets.flatMap(set => [...set])))); -} - -function loadCachedFacts( - store: CodeGraphStoreShape, - databasePath: string, - files: readonly CodeGraphInventoryFile[], - languagePacks: CodeGraphLanguagePackRegistryShape, -): Effect.Effect< - { - readonly bytes: number; - readonly bytesByPath: ReadonlyMap; - readonly facts: ReadonlyMap; - }, - unknown -> { - return Effect.forEach( - groupFilesByCacheIdentity(files, languagePacks), - group => - Effect.gen(function* () { - const active = yield* store.loadCachedFacts(databasePath, group.files, group.cacheIdentity); - const missing = group.files.filter(file => !active.facts.has(file.path)); - if (missing.length === 0) return active; - const degraded = yield* store.loadCachedFacts( - databasePath, - missing, - degradedParserCacheIdentity(group.cacheIdentity), - ); - return { - bytes: active.bytes + degraded.bytes, - bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), - facts: new Map([...active.facts, ...degraded.facts]), - }; - }), - {concurrency: 1}, - ).pipe( - Effect.map(groups => { - const output = new Map(); - const bytesByPath = new Map(); - let bytes = 0; - for (const group of groups) { - for (const [path, facts] of group.facts) { - const persistedBytes = group.bytesByPath?.get(path); - if (persistedBytes !== undefined && persistedBytes <= CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM) { - output.set(path, facts); - bytesByPath.set(path, persistedBytes); - bytes += persistedBytes; - continue; - } - const budgeted = budgetCachedCodeGraphFacts(facts); - const budgetedBytes = cachedCodeGraphFactBytes(budgeted); - output.set(path, budgeted); - bytesByPath.set(path, budgetedBytes); - bytes += budgetedBytes; - } - } - return {bytes, bytesByPath, facts: output}; - }), - ); -} - -function loadCachedFactsWithPackProvenance( - store: CodeGraphStoreShape, - databasePath: string, - files: readonly CodeGraphInventoryFile[], - languagePacks: CodeGraphLanguagePackRegistryShape, - provenance: readonly CodeGraphLanguagePackProvenance[], -): Effect.Effect< - { - readonly bytes: number; - readonly bytesByPath: ReadonlyMap; - readonly facts: ReadonlyMap; - }, - unknown -> { - const provenanceById = new Map(provenance.map(pack => [pack.id, pack])); - const groups = new Map(); - let unmatched = false; - for (const file of files) { - const match = Option.getOrUndefined(languagePacks.match(file.path)); - const identity = match === undefined ? undefined : provenanceById.get(match.pack.id)?.cacheIdentity; - if (identity === undefined) { - unmatched = true; - continue; - } - const group = groups.get(identity) ?? []; - group.push(file); - groups.set(identity, group); - } - if (unmatched) return Effect.succeed({bytes: 0, bytesByPath: new Map(), facts: new Map()}); - return Effect.forEach( - [...groups], - ([cacheIdentity, groupFiles]) => - Effect.gen(function* () { - const active = yield* store.loadCachedFacts(databasePath, groupFiles, cacheIdentity); - const missing = groupFiles.filter(file => !active.facts.has(file.path)); - if (missing.length === 0) return active; - const degraded = yield* store.loadCachedFacts( - databasePath, - missing, - degradedParserCacheIdentity(cacheIdentity), - ); - return { - bytes: active.bytes + degraded.bytes, - bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), - facts: new Map([...active.facts, ...degraded.facts]), - }; - }), - {concurrency: 1}, - ).pipe( - Effect.map(loaded => { - const facts = new Map(); - const bytesByPath = new Map(); - let bytes = 0; - for (const group of loaded) { - for (const [path, fact] of group.facts) { - const persistedBytes = group.bytesByPath?.get(path); - if (persistedBytes !== undefined && persistedBytes <= CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM) { - facts.set(path, fact); - bytesByPath.set(path, persistedBytes); - bytes += persistedBytes; - continue; - } - const budgeted = budgetCachedCodeGraphFacts(fact); - const budgetedBytes = cachedCodeGraphFactBytes(budgeted); - facts.set(path, budgeted); - bytesByPath.set(path, budgetedBytes); - bytes += budgetedBytes; - } - } - return {bytes, bytesByPath, facts}; - }), - ); -} - -function cachedFactsMetadata( - store: CodeGraphStoreShape, - databasePath: string, - files: readonly CodeGraphInventoryFile[], - languagePacks: CodeGraphLanguagePackRegistryShape, -): Effect.Effect< - {readonly bytes: number; readonly bytesByPath: ReadonlyMap; readonly files: number}, - unknown -> { - return Effect.forEach( - groupFilesByCacheIdentity(files, languagePacks), - group => - Effect.gen(function* () { - const active = yield* store.loadCachedFacts(databasePath, group.files, group.cacheIdentity, {decode: false}); - const activeKeys = active.keys ?? new Set(active.facts.keys()); - const missing = group.files.filter(file => !activeKeys.has(file.path)); - if (missing.length === 0) - return {bytes: active.bytes, bytesByPath: active.bytesByPath ?? new Map(), keys: activeKeys}; - const degraded = yield* store.loadCachedFacts( - databasePath, - missing, - degradedParserCacheIdentity(group.cacheIdentity), - {decode: false}, - ); - const degradedKeys = degraded.keys ?? new Set(degraded.facts.keys()); - return { - bytes: active.bytes + degraded.bytes, - bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), - keys: new Set([...activeKeys, ...degradedKeys]), - }; - }), - {concurrency: 1}, - ).pipe( - Effect.map(groups => { - const bytesByPath = new Map( - groups.flatMap(group => [...group.bytesByPath]).map(([path, bytes]) => [path, bytes] as const), - ); - return { - bytes: [...bytesByPath.values()].reduce((total, bytes) => total + bytes, 0), - bytesByPath, - files: new Set(groups.flatMap(group => [...group.keys])).size, - }; - }), - ); -} - -function groupFilesByCacheIdentity( - files: readonly T[], - languagePacks: CodeGraphLanguagePackRegistryShape, -): readonly {readonly cacheIdentity: string; readonly files: readonly T[]}[] { - const groups = new Map(); - for (const file of files) { - const matched = languagePacks.cacheIdentityForPath(file.path); - const identity = matched._tag === 'Some' ? matched.value : 'unmatched'; - const group = groups.get(identity); - if (group) group.push(file); - else groups.set(identity, [file]); - } - return [...groups] - .sort(([left], [right]) => compareCodeUnits(left, right)) - .map(([cacheIdentity, groupedFiles]) => ({cacheIdentity, files: groupedFiles})); -} +export {CodeGraphIndexer} from './indexer_service.js'; +export { + codeGraphIndexEnsuresVectors, + type CodeGraphCommitLease, + type CodeGraphIndexerShape, + type CodeGraphIndexOptions, + type DirectPersistentCapacityProtection, +} from './indexer_types.js'; +export { + createCachedCodeGraphFactsAttributor, + deriveCachedCodeGraphFacts, + hasSameCodeGraphResolutionSurface, + resolvePersistedReexportTerminals, + reusableBaseFileSetFingerprint, +} from './indexer_incremental.js'; +export { + addMaterializationRows, + cacheContentBatch, + codeGraphActiveParserCacheKey, + codeGraphDirectPersistentCapacityProtector, + codeGraphParserCacheLookupGenerations, + deduplicateMaterializationRelationships, + directFullSnapshotIdentity, + estimatedMaterializationStorageBytes, + extractorSetIdentity, + extractorSetIdentityFromPackProvenance, + factMaterializationBatches, + graphContentIdentity, + materializationRowsWithStoreProgress, + materializationStoragePlan, + materializationStorageShortfalls, + parserCacheIdentity, + persistentMaterializationTransactionBatches, + shouldReuseReadySnapshotForCleanCommit, + snapshotIdentity, + type CodeGraphCacheContentCoalescer, + type CodeGraphCacheExtractedRow, + type DirectPersistentCapacityContext, + type MaterializationStorageAvailability, + type MaterializationStoragePlan, + type PersistentMaterializationTransactionCandidate, +} from './indexer_materialization.js'; diff --git a/src/code_graph/indexer_build.ts b/src/code_graph/indexer_build.ts new file mode 100644 index 00000000..a3a3691f --- /dev/null +++ b/src/code_graph/indexer_build.ts @@ -0,0 +1,1784 @@ +import {Clock, Effect, FileSystem, Option, Path} from 'effect'; +import {sha256HexSync} from '../crypto/sha256.js'; +import {withExclusiveFileLock} from '../effect/file_lock.js'; +import {SystemInfo} from '../effect/system.js'; +import {withThreadnoteProcessActivity} from '../process_diagnostics.js'; +import type {CodeGraphBuildOwnerIdentity} from './build_owner.js'; +import {readCodeGraphBuildStatuses} from './build_status.js'; +import {canonicalCodeGraphMonikers} from './cross_repository/monikers.js'; +import type {CodeGraphMonikerV1} from './cross_repository/types.js'; +import {isCodeGraphCapacityPause} from './disk_capacity.js'; +import type {CodeGraphEmbeddingIndexShape, CodeGraphEmbeddingStatus} from './embedding.js'; +import {finalCodeGraphFactBatches, serializeBoundedCodeGraphFact} from './fact_budget.js'; +import { + assessIncrementalOverlay, + assessReusableCleanBaseCompatibility, + createCachedCodeGraphFactsAttributor, + overlayFallbackDescription, + reusableBaseFileSetFingerprint, +} from './indexer_incremental.js'; +import { + CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS, + CODE_GRAPH_LOCK_OPTIONS, + PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES, + PERSISTENT_MATERIALIZATION_TRANSACTION_FILES, + PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES, + addMaterializationRows, + cachedFactsMetadata, + codeGraphDirectPersistentCapacityProtector, + deduplicateMaterializationRelationships, + embeddingSymbolSource, + estimatedMaterializationStorageBytes, + extractorSetIdentity, + extractorSetIdentityFromPackProvenance, + factMaterializationBatches, + forcedSnapshotIdentity, + graphContentIdentity, + loadCachedFacts, + materializationRows, + materializationRowsWithStoreProgress, + materializationStagingStage, + materializationStorageFiles, + materializationStoragePlan, + materializationStorageShortfalls, + messageOf, + persistentMaterializationTransactionBatches, + promoteReadySnapshotWithCapacity, + reusableReadySnapshotForCleanCommit, + snapshotIdentity, + uniqueById, + verifyIndexInput, + type PersistentMaterializationTransactionCandidate, +} from './indexer_materialization.js'; +import {CodeGraphIndexOperationError, codeGraphInventoryFileChanged, sameInventoryPaths} from './indexer_shared.js'; +import type { + CodeGraphIndexOptions, + CommittedBaseResult, + DirectPersistentCapacityProtection, + IncrementalOverlayAssessment, + IncrementalOverlayPreassessment, + ReusableCleanSnapshotAttempt, +} from './indexer_types.js'; +import {preferredIncrementalBaseCommitGroups} from './incremental_base_selection.js'; +import type {CodeGraphInventory} from './inventory.js'; +import {assessCodeGraphLanguagePackDelta} from './languages/provenance.js'; +import {packDerivationIdentity, type CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; +import type {CodeGraphWorkspace} from './languages/types.js'; +import {codeGraphRequestBuildLockPath, codeGraphSnapshotBuildLockPath, type CodeGraphLayout} from './layout.js'; +import {compareCodeUnits} from './ordering.js'; +import { + CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION, + materializedShardDerivationIdentity, + type CodeGraphDirectPersistentCapacityProtector, + type CodeGraphRetiredSnapshotCleanupProgress, + type CodeGraphReusableCleanBase, + type CodeGraphStagingProgress, + type CodeGraphStoreShape, +} from './store.js'; +import { + CODE_GRAPH_EXTRACTOR_SET_VERSION, + type CodeGraphEdge, + type CodeGraphIndexSummary, + type CodeGraphInventoryFile, + type CodeGraphMaterializationActivity, + type CodeGraphMaterializationMetrics, + type CodeGraphMaterializationRows, + type CodeGraphOverlayFallbackReason, + type CodeGraphProgress, + type CodeGraphReference, + type CodeGraphSnapshot, + type CodeGraphSymbol, + type RepositoryIdentity, +} from './types.js'; + +export function withCodeGraphProcessLock( + fs: FileSystem.FileSystem, + lockPath: string, + onContention: () => Effect.Effect, + builderOperation: string, + effect: Effect.Effect, +) { + return withThreadnoteProcessActivity( + 'graph-waiter', + 'repository-lock', + withExclusiveFileLock( + fs, + lockPath, + {...CODE_GRAPH_LOCK_OPTIONS, onContention}, + withThreadnoteProcessActivity('graph-builder', builderOperation, effect), + ), + ); +} + +export function writerSessionOptions(layout: CodeGraphLayout, options: CodeGraphIndexOptions) { + return { + cleanupCompletedBuildRows: true, + ...(options.onSqliteWriterConfigured ? {onSqliteWriterConfigured: options.onSqliteWriterConfigured} : {}), + onWriterContention: () => + (options.onProgress?.({phase: 'waiting', reason: 'database-writer'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + ...(options.sqliteWriterTuning ? {sqliteWriterTuning: options.sqliteWriterTuning} : {}), + writerLockPath: layout.databaseWriteLockPath, + } as const; +} + +export function retiredSnapshotCleanupReporter(onProgress: CodeGraphIndexOptions['onProgress']) { + return (progress: CodeGraphRetiredSnapshotCleanupProgress) => + ( + onProgress?.({ + completed: progress.snapshotsCompleted, + pagesCompleted: progress.pagesCompleted, + phase: 'reclaiming', + rowsDeleted: progress.rowsDeleted, + total: progress.snapshotsTotal, + unit: 'snapshots', + }) ?? Effect.void + ).pipe(Effect.catch(() => Effect.void)); +} + +export function withSharedCleanRequestGate(input: { + readonly checkoutId: string; + readonly effect: Effect.Effect; + readonly fs: FileSystem.FileSystem; + readonly onProgress: CodeGraphIndexOptions['onProgress']; + readonly path: Path.Path; + readonly requestedOverlay: {readonly dirty: boolean; readonly fingerprint?: string} | undefined; + readonly requestKey: string | undefined; + readonly threadnoteHome: string; +}) { + if (!input.requestKey || input.requestedOverlay?.dirty !== false) return input.effect; + return withExclusiveFileLock( + input.fs, + codeGraphRequestBuildLockPath(input.path, input.threadnoteHome, input.checkoutId, input.requestKey), + { + ...CODE_GRAPH_LOCK_OPTIONS, + onContention: () => + (input.onProgress?.({phase: 'waiting', reason: 'request-lock'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + }, + input.effect, + ); +} + +export const completedConcurrentSnapshot = Effect.fn('codeGraph.completedConcurrentSnapshot')(function* ( + store: CodeGraphStoreShape, + layout: CodeGraphLayout, + identity: RepositoryIdentity, + overlay: {readonly dirty: boolean; readonly fingerprint?: string}, + requestKey: string, + requireDirectFull: boolean, +) { + const statuses = yield* readCodeGraphBuildStatuses(layout); + const completed = statuses.find( + status => status.state === 'completed' && status.request?.key === requestKey && status.result?.snapshotId, + ); + if (!completed?.result?.snapshotId) return undefined; + const ready = yield* store.currentLexicalReadySnapshotById(layout.databasePath, completed.result.snapshotId); + if ( + !ready || + ready.commit !== identity.headCommit || + ready.dirty !== overlay.dirty || + (overlay.dirty && requireDirectFull && (ready.baseSnapshotId !== undefined || !ready.id.endsWith('-direct'))) + ) { + return undefined; + } + return ready; +}); + +export const prepareReadyAnalysisSummary = Effect.fn('codeGraph.prepareReadyAnalysisSummary')(function* (input: { + readonly databasePath: string; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly snapshotId: string; + readonly store: CodeGraphStoreShape; +}) { + yield* input.onProgress?.({ + phase: 'activating', + snapshotId: input.snapshotId, + subphase: 'summarizing-analysis', + }) ?? Effect.void; + return yield* ( + typeof input.store.ensureAnalysisSummary === 'function' + ? input.store.ensureAnalysisSummary(input.databasePath, input.snapshotId) + : Effect.succeed(false) + ).pipe( + Effect.ensuring( + ( + input.onProgress?.({phase: 'activating', snapshotId: input.snapshotId, subphase: 'complete'}) ?? Effect.void + ).pipe(Effect.catch(() => Effect.void)), + ), + ); +}); + +export const reuseReadySnapshot = Effect.fn('codeGraph.reuseReadySnapshot')(function* (input: { + readonly embedding: CodeGraphEmbeddingIndexShape; + readonly ensureVectors: boolean; + readonly identity: RepositoryIdentity; + readonly layout: CodeGraphLayout; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly reusedFiles: number; + readonly skippedFiles: number; + readonly snapshot: CodeGraphSnapshot; + readonly startedAt: number; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; + readonly totalFiles: number; +}) { + yield* input.onProgress?.({phase: 'activating', snapshotId: input.snapshot.id, subphase: 'structural-ready'}) ?? + Effect.void; + let analysisSummaryFailure: string | undefined; + const analysisSummaryBackfilled = input.snapshot.dirty + ? yield* ( + input.onProgress?.({phase: 'activating', snapshotId: input.snapshot.id, subphase: 'complete'}) ?? Effect.void + ).pipe(Effect.as(false)) + : yield* prepareReadyAnalysisSummary({ + databasePath: input.layout.databasePath, + onProgress: input.onProgress, + snapshotId: input.snapshot.id, + store: input.store, + }).pipe( + Effect.catch(cause => + Effect.sync(() => { + analysisSummaryFailure = messageOf(cause); + return false; + }), + ), + ); + const diagnostics: string[] = analysisSummaryBackfilled + ? ['Built the persisted whole-graph analysis summary for this reused snapshot.'] + : analysisSummaryFailure + ? [`Whole-graph analysis summary will be retried lazily: ${analysisSummaryFailure}`] + : []; + if (!input.ensureVectors) { + const vectorCheck = yield* input.embedding + .check(input.threadnoteHome, input.layout, input.snapshot.id) + .pipe(Effect.catch(cause => Effect.succeed({reason: messageOf(cause), state: 'unavailable'} as const))); + if (vectorCheck.state !== 'ready') { + diagnostics.push( + `Vector graph retrieval unavailable: ${vectorCheck.reason ?? 'deferred until an explicit vector refresh'}`, + ); + } + return { + diagnostics, + durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, + identity: input.identity, + materialization: { + mode: 'reused-snapshot', + stagedFiles: 0, + totalFiles: input.totalFiles, + }, + reusedFiles: input.reusedFiles, + skippedFiles: input.skippedFiles, + snapshot: input.snapshot, + } satisfies CodeGraphIndexSummary; + } + const vectorCheck = yield* input.embedding + .check(input.threadnoteHome, input.layout, input.snapshot.id) + .pipe(Effect.catch(cause => Effect.succeed({reason: messageOf(cause), state: 'unavailable'} as const))); + const symbols = + vectorCheck.state === 'ready' + ? [] + : embeddingSymbolSource(input.store, input.layout.databasePath, input.snapshot.id); + const repaired = yield* input.embedding + .ensure(input.threadnoteHome, input.layout, input.snapshot, symbols, { + onProgress: input.onProgress, + }) + .pipe( + Effect.catch(cause => + Effect.succeed({ + embedded: 0, + ready: false, + reason: messageOf(cause), + reused: 0, + } satisfies CodeGraphEmbeddingStatus), + ), + ); + if (!repaired.ready) { + diagnostics.push(`Vector graph retrieval unavailable: ${repaired.reason ?? 'unknown reason'}`); + } + return { + diagnostics, + durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, + identity: input.identity, + materialization: { + mode: 'reused-snapshot', + stagedFiles: 0, + totalFiles: input.totalFiles, + }, + reusedFiles: input.reusedFiles, + skippedFiles: input.skippedFiles, + snapshot: input.snapshot, + } satisfies CodeGraphIndexSummary; +}); + +export function codeGraphBuildRequestKey( + identity: Pick, + overlay: {readonly dirty: boolean; readonly fingerprint?: string}, + languagePacks: CodeGraphLanguagePackRegistryShape, + incrementalOverlay: boolean | undefined, +): string { + const parserIdentities = languagePacks.cacheIdentities.join('\n'); + const derivationIdentities = languagePacks.packs.map(packDerivationIdentity).sort(compareCodeUnits).join('\n'); + return sha256HexSync( + [ + 'code-graph-build-request-v3', + CODE_GRAPH_EXTRACTOR_SET_VERSION, + `lexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}`, + identity.repositoryId, + identity.checkoutId, + overlay.dirty ? identity.worktreeId : 'shared-commit', + identity.headCommit, + overlay.dirty ? (overlay.fingerprint ?? 'dirty-without-fingerprint') : 'clean', + overlay.dirty && incrementalOverlay === false ? 'direct-full' : 'default', + 'ignore-policy:3', + parserIdentities, + derivationIdentities, + ].join('\n'), + ); +} + +export const buildOwnedCleanSnapshot = Effect.fn('codeGraph.buildOwnedCleanSnapshot')(function* (input: { + readonly buildOwner: CodeGraphBuildOwnerIdentity; + readonly capacityProtection: DirectPersistentCapacityProtection; + readonly embedding: CodeGraphEmbeddingIndexShape; + readonly ensureVectors: boolean; + readonly existing: CodeGraphSnapshot | undefined; + readonly fallbackSnapshotId: string; + readonly force: boolean; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly logicalSnapshotId: string; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; + readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; + readonly startedAt: number; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; +}) { + return yield* withExclusiveFileLock( + input.fs, + codeGraphSnapshotBuildLockPath( + yield* Path.Path, + input.threadnoteHome, + input.identity.checkoutId, + input.logicalSnapshotId, + ), + { + ...CODE_GRAPH_LOCK_OPTIONS, + onContention: () => + (input.onProgress?.({phase: 'waiting', reason: 'snapshot-build'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + }, + Effect.gen(function* () { + let cleanFallbackAssessment: IncrementalOverlayAssessment | undefined; + if (!input.force) { + const ready = yield* input.store.currentLexicalReadySnapshotById( + input.layout.databasePath, + input.logicalSnapshotId, + ); + if (ready) { + if (input.existing?.id !== ready.id) { + yield* promoteReadySnapshotWithCapacity(input, ready.id); + } + return yield* reuseReadySnapshot({ + embedding: input.embedding, + ensureVectors: input.ensureVectors, + identity: input.identity, + layout: input.layout, + onProgress: input.onProgress, + reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, + skippedFiles: input.inventory.skipped, + snapshot: ready, + startedAt: input.startedAt, + store: input.store, + threadnoteHome: input.threadnoteHome, + totalFiles: input.inventory.files.length, + }); + } + const extractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); + const graphContentId = graphContentIdentity(extractorSet, input.inventory.files); + const commitReady = yield* reusableReadySnapshotForCleanCommit({ + databasePath: input.layout.databasePath, + extractorSet, + graphContentId, + headCommit: input.identity.headCommit, + repositoryId: input.identity.repositoryId, + store: input.store, + }); + if (commitReady) { + if (input.existing?.id !== commitReady.id) { + yield* promoteReadySnapshotWithCapacity(input, commitReady.id); + } + return yield* reuseReadySnapshot({ + embedding: input.embedding, + ensureVectors: input.ensureVectors, + identity: input.identity, + layout: input.layout, + onProgress: input.onProgress, + reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, + skippedFiles: input.inventory.skipped, + snapshot: commitReady, + startedAt: input.startedAt, + store: input.store, + threadnoteHome: input.threadnoteHome, + totalFiles: input.inventory.files.length, + }); + } + const workspace = + input.inventory.workspace ?? (yield* input.languagePacks.discoverWorkspace(input.inventory.files)); + const reused = yield* attemptReusableCleanSnapshot(input, workspace); + if (Option.isSome(reused)) { + if (reused.value.mode === 'complete') return reused.value.summary; + cleanFallbackAssessment = {mode: 'fallback', reason: reused.value.reason}; + } + } + const resumed = input.force + ? yield* input.store.resumableForcedBuild(input.layout.databasePath, input.logicalSnapshotId) + : undefined; + const building: CodeGraphSnapshot = resumed ?? { + commit: input.identity.headCommit, + dirty: false, + edgeCount: 0, + extractorSet: extractorSetIdentity(input.inventory.files, input.languagePacks), + fileCount: 0, + graphContentId: graphContentIdentity( + extractorSetIdentity(input.inventory.files, input.languagePacks), + input.inventory.files, + ), + id: input.fallbackSnapshotId, + repositoryId: input.identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: input.identity.worktreeId, + }; + const ownerToken = yield* input.store.claimPersistentBuild(input.layout.databasePath, input.identity, building, { + logicalSnapshotId: input.logicalSnapshotId, + owner: input.buildOwner, + }); + return yield* buildAndActivate({ + activatePointer: true, + building, + capacityProtection: input.capacityProtection, + embedding: input.embedding, + ensureVectors: input.ensureVectors, + existing: input.existing, + force: input.force, + fs: input.fs, + identity: input.identity, + incrementalAssessment: cleanFallbackAssessment, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + onProgress: input.onProgress, + persistentMaterializationTransactionBatchLimit: input.persistentMaterializationTransactionBatchLimit, + persistentOwnerToken: ownerToken, + requestedOverlay: input.requestedOverlay, + startedAt: input.startedAt, + store: input.store, + threadnoteHome: input.threadnoteHome, + }).pipe( + Effect.catch(cause => + isCodeGraphCapacityPause(cause) + ? Effect.fail(cause) + : input.store + .markFailed(input.layout.databasePath, building.id, messageOf(cause), ownerToken) + .pipe(Effect.andThen(Effect.fail(cause))), + ), + ); + }), + ); +}); + +const attemptReusableCleanSnapshot = Effect.fn('codeGraph.attemptReusableCleanSnapshot')(function* ( + input: { + readonly capacityProtection: DirectPersistentCapacityProtection; + readonly embedding: CodeGraphEmbeddingIndexShape; + readonly ensureVectors: boolean; + readonly existing: CodeGraphSnapshot | undefined; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly logicalSnapshotId: string; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; + readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; + readonly startedAt: number; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; + }, + workspace: CodeGraphWorkspace, +) { + if (!input.store.reusableCleanBase || !input.store.activateCleanSnapshotAlias) { + return Option.none(); + } + const extractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); + const preferredCommitGroups = yield* preferredIncrementalBaseCommitGroups( + input.identity.repoRoot, + input.identity.headCommit, + ); + const candidate = yield* input.store.reusableCleanBase( + input.layout.databasePath, + input.identity.repositoryId, + extractorSet, + workspace.fingerprint, + reusableBaseFileSetFingerprint(input.inventory.files), + graphContentIdentity(extractorSet, input.inventory.files), + preferredCommitGroups, + true, + ); + if (!candidate || candidate.snapshot.id === input.logicalSnapshotId) + return Option.none(); + const baseByPath = new Map(candidate.files.map(file => [file.path, file])); + if (input.inventory.files.some(file => file.source !== 'commit')) { + return Option.none(); + } + const lease = yield* input.store + .acquireSnapshotLease(input.layout.databasePath, candidate.snapshot.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) + .pipe(Effect.option); + if (Option.isNone(lease)) return Option.none(); + return yield* Effect.acquireUseRelease( + Effect.succeed(lease.value), + () => + Effect.gen(function* () { + const packDelta = + candidate.snapshot.extractorSet === extractorSet + ? ({changedPackIds: [], mode: 'compatible'} as const) + : assessCodeGraphLanguagePackDelta( + candidate.receipt.packProvenance, + input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), + ); + if ( + packDelta.mode === 'fallback' || + (candidate.snapshot.extractorSet !== extractorSet && + candidate.snapshot.extractorSet !== + extractorSetIdentityFromPackProvenance(candidate.receipt.packProvenance)) + ) { + return Option.some({mode: 'fallback', reason: 'extractor-context-changed'}); + } + const changedPackIds = new Set(packDelta.changedPackIds); + const modifiedFiles = input.inventory.files.filter(file => { + const base = baseByPath.get(file.path); + return ( + !base || + base.contentHash !== file.contentHash || + base.language !== file.language || + base.mode !== file.mode || + base.size !== file.size || + Option.match(input.languagePacks.match(file.path), { + onNone: () => false, + onSome: match => changedPackIds.has(match.pack.id), + }) + ); + }); + const currentPaths = new Set(input.inventory.files.map(file => file.path)); + const deletedPaths = candidate.files.filter(file => !currentPaths.has(file.path)).map(file => file.path); + if ( + modifiedFiles.length === 0 && + deletedPaths.length === 0 && + candidate.snapshot.extractorSet === extractorSet + ) { + const alias: CodeGraphSnapshot = { + baseSnapshotId: candidate.snapshot.id, + commit: input.identity.headCommit, + dirty: false, + edgeCount: candidate.snapshot.edgeCount, + extractorSet, + fileCount: candidate.snapshot.fileCount, + graphContentId: graphContentIdentity(extractorSet, input.inventory.files), + id: input.logicalSnapshotId, + repositoryId: input.identity.repositoryId, + state: 'ready', + symbolCount: candidate.snapshot.symbolCount, + worktreeId: input.identity.worktreeId, + }; + yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); + yield* input.store.activateCleanSnapshotAlias!( + input.layout.databasePath, + input.identity, + alias, + candidate.snapshot.id, + ); + yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); + yield* promoteReadySnapshotWithCapacity(input, alias.id); + yield* verifyIndexInput(input.identity, true, input.threadnoteHome, input.requestedOverlay); + return Option.some({ + mode: 'complete', + summary: yield* reuseReadySnapshot({ + embedding: input.embedding, + ensureVectors: input.ensureVectors, + identity: input.identity, + layout: input.layout, + onProgress: input.onProgress, + reusedFiles: input.inventory.files.length, + skippedFiles: input.inventory.skipped, + snapshot: alias, + startedAt: input.startedAt, + store: input.store, + threadnoteHome: input.threadnoteHome, + totalFiles: input.inventory.files.length, + }), + }); + } + const assessmentInput = { + candidate, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector(input), + store: input.store, + }; + const sameFileSet = deletedPaths.length === 0 && modifiedFiles.every(file => baseByPath.has(file.path)); + const boundedAssessment = sameFileSet + ? yield* assessReusableCleanBaseCompatibility(assessmentInput, workspace, modifiedFiles) + : ({mode: 'fallback', reason: 'file-set-changed'} as const); + if (boundedAssessment.mode === 'fallback') { + return Option.some(boundedAssessment); + } + const preassessment = boundedAssessment; + const committedBase: CommittedBaseResult = { + diagnostics: [], + leaseToken: Option.none(), + snapshot: candidate.snapshot, + stagingReusable: false, + }; + const building: CodeGraphSnapshot = { + baseSnapshotId: candidate.snapshot.id, + commit: input.identity.headCommit, + dirty: false, + edgeCount: 0, + extractorSet, + fileCount: 0, + graphContentId: graphContentIdentity(extractorSet, input.inventory.files), + id: input.logicalSnapshotId, + repositoryId: input.identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: input.identity.worktreeId, + }; + const incrementalAssessment = yield* assessIncrementalOverlay( + { + building, + committedBase, + force: false, + incrementalOverlayEnabled: true, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + store: input.store, + }, + workspace, + preassessment, + ); + if (incrementalAssessment.mode === 'fallback') { + return Option.some(incrementalAssessment); + } + yield* input.onProgress?.({ + completed: 0, + phase: 'materializing', + reused: input.inventory.files.length - incrementalAssessment.files.length, + total: incrementalAssessment.files.length, + unit: 'files', + }) ?? Effect.void; + const prepared = yield* input.store.preparePersistedIncrementalActivation( + input.layout.databasePath, + candidate.snapshot.id, + incrementalAssessment.files, + incrementalAssessment.facts, + { + deletedPaths: incrementalAssessment.deletedPaths, + resolutionClosure: incrementalAssessment.resolutionClosure, + }, + assessmentInput.persistentCapacityProtector, + ); + if (!prepared) { + return Option.some({mode: 'fallback', reason: 'staging-identity-mismatch'}); + } + yield* input.store.markBuilding(input.layout.databasePath, input.identity, building); + const summary = yield* buildAndActivate({ + activatePointer: true, + building, + capacityProtection: input.capacityProtection, + committedBase, + embedding: input.embedding, + ensureVectors: input.ensureVectors, + existing: input.existing, + force: false, + fs: input.fs, + identity: input.identity, + incrementalAssessment, + incrementalOverlayEnabled: true, + incrementalPrepared: true, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + onProgress: input.onProgress, + persistentMaterializationTransactionBatchLimit: input.persistentMaterializationTransactionBatchLimit, + requestedOverlay: input.requestedOverlay, + startedAt: input.startedAt, + store: input.store, + threadnoteHome: input.threadnoteHome, + workspace, + }).pipe( + Effect.catch(cause => + input.store + .markFailed(input.layout.databasePath, building.id, messageOf(cause)) + .pipe(Effect.andThen(Effect.fail(cause))), + ), + ); + return Option.some({mode: 'complete', summary}); + }), + token => input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), + ); +}); + +export const attemptReusableDirtyBase = Effect.fn('codeGraph.attemptReusableDirtyBase')(function* ( + input: { + readonly extractorSet: string; + readonly identity: RepositoryIdentity; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; + readonly store: CodeGraphStoreShape; + }, + workspace: CodeGraphWorkspace, +) { + if (!input.store.reusableCleanBase) { + return Option.none<{ + readonly committedBase: CommittedBaseResult; + readonly preassessment: Extract; + }>(); + } + // Prefer the exact committed snapshot path below when it is itself a root + // reusable base. A clean incremental snapshot is already layered, so another + // overlay must instead reuse its root and include the cumulative changed set. + const committedExtractorSet = extractorSetIdentity(input.inventory.committedFiles, input.languagePacks); + const exactCommittedSnapshotId = snapshotIdentity( + input.identity, + false, + committedExtractorSet, + input.inventory.committedFiles, + ); + const exactCommittedSnapshot = yield* input.store.currentLexicalReadySnapshotById( + input.layout.databasePath, + exactCommittedSnapshotId, + ); + if (exactCommittedSnapshot && exactCommittedSnapshot.baseSnapshotId === undefined) { + return Option.none(); + } + const committedFileSetFingerprint = reusableBaseFileSetFingerprint(input.inventory.committedFiles); + const committedGraphContentId = graphContentIdentity(committedExtractorSet, input.inventory.committedFiles); + const commitReady = yield* input.store.readySnapshotForCommit( + input.layout.databasePath, + input.identity.repositoryId, + input.identity.headCommit, + committedExtractorSet, + ); + const commitReceipt = commitReady + ? yield* input.store.reusableBaseReceipt(input.layout.databasePath, commitReady.id) + : undefined; + let candidate: CodeGraphReusableCleanBase | undefined = + commitReady && + commitReceipt && + commitReady.graphContentId === committedGraphContentId && + commitReceipt.fileSetFingerprint === committedFileSetFingerprint && + commitReceipt.workspaceFingerprint === workspace.fingerprint + ? {files: input.inventory.committedFiles, receipt: commitReceipt, snapshot: commitReady} + : undefined; + if (!candidate) { + const preferredCommitGroups = yield* preferredIncrementalBaseCommitGroups( + input.identity.repoRoot, + input.identity.headCommit, + ); + candidate = yield* input.store.reusableCleanBase( + input.layout.databasePath, + input.identity.repositoryId, + input.extractorSet, + workspace.fingerprint, + reusableBaseFileSetFingerprint(input.inventory.files), + graphContentIdentity(input.extractorSet, input.inventory.files), + preferredCommitGroups, + true, + ); + } + if (!candidate) return Option.none(); + const lease = yield* input.store + .acquireSnapshotLease(input.layout.databasePath, candidate.snapshot.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) + .pipe(Effect.option); + if (Option.isNone(lease)) return Option.none(); + const leaseToken = yield* Effect.acquireRelease(Effect.succeed(lease.value), token => + input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), + ); + const packDelta = + candidate.snapshot.extractorSet === input.extractorSet + ? ({changedPackIds: [], mode: 'compatible'} as const) + : assessCodeGraphLanguagePackDelta( + candidate.receipt.packProvenance, + input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), + ); + if ( + packDelta.mode === 'fallback' || + (candidate.snapshot.extractorSet !== input.extractorSet && + candidate.snapshot.extractorSet !== extractorSetIdentityFromPackProvenance(candidate.receipt.packProvenance)) + ) { + return Option.none(); + } + const changedPackIds = new Set(packDelta.changedPackIds); + const alignedCommitCandidate = sameInventoryPaths(candidate.files, input.inventory.files); + const baseByPath = alignedCommitCandidate ? undefined : new Map(candidate.files.map(file => [file.path, file])); + const currentPaths = alignedCommitCandidate ? undefined : new Set(input.inventory.files.map(file => file.path)); + const modifiedFiles = input.inventory.files.filter((file, index) => { + const base = alignedCommitCandidate ? candidate.files[index] : baseByPath!.get(file.path); + return codeGraphInventoryFileChanged(base, file, input.languagePacks, changedPackIds); + }); + const deletedPaths = alignedCommitCandidate + ? [] + : candidate.files.filter(file => !currentPaths!.has(file.path)).map(file => file.path); + if (modifiedFiles.length === 0 && deletedPaths.length === 0) return Option.none(); + const assessmentInput = { + candidate, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + persistentCapacityProtector: input.persistentCapacityProtector, + store: input.store, + }; + const sameFileSet = + alignedCommitCandidate || (deletedPaths.length === 0 && modifiedFiles.every(file => baseByPath!.has(file.path))); + const boundedAssessment = sameFileSet + ? yield* assessReusableCleanBaseCompatibility(assessmentInput, workspace, modifiedFiles) + : ({mode: 'fallback', reason: 'file-set-changed'} as const); + if (boundedAssessment.mode === 'fallback') return Option.none(); + const preassessment = boundedAssessment; + return Option.some({ + committedBase: { + diagnostics: [ + `Dirty snapshot reused compatible persisted base ${candidate.snapshot.id} without first building commit ${input.identity.headCommit}.`, + ], + leaseToken: Option.some(leaseToken), + snapshot: candidate.snapshot, + stagingReusable: false, + }, + preassessment, + }); +}); + +export const ensureCommittedBase = Effect.fn('codeGraph.ensureCommittedBase')(function* (input: { + readonly buildOwner: CodeGraphBuildOwnerIdentity; + readonly capacityProtection: DirectPersistentCapacityProtection; + readonly embedding: CodeGraphEmbeddingIndexShape; + readonly existing?: CodeGraphSnapshot; + readonly force: boolean; + readonly forceGeneration?: string; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; + readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; + readonly startedAt: number; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; +}) { + const cleanInventory: CodeGraphInventory = { + committedFiles: input.inventory.committedFiles, + committedParsedFiles: input.inventory.committedParsedFiles, + dirty: false, + files: input.inventory.committedFiles, + parsedFiles: input.inventory.committedParsedFiles, + skipped: input.inventory.skipped, + }; + const extractorSet = extractorSetIdentity(cleanInventory.files, input.languagePacks); + const logicalSnapshotId = snapshotIdentity(input.identity, false, extractorSet, cleanInventory.files); + const snapshotId = forcedSnapshotIdentity(logicalSnapshotId, input.forceGeneration); + const existing = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, snapshotId); + if (existing) { + const lease = yield* input.store + .acquireSnapshotLease(input.layout.databasePath, existing.id, CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS) + .pipe(Effect.option); + if (Option.isSome(lease)) { + const leaseToken = yield* Effect.acquireRelease(Effect.succeed(lease.value), token => + input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), + ); + return { + diagnostics: [], + leaseToken: Option.some(leaseToken), + snapshot: existing, + stagingReusable: false, + } satisfies CommittedBaseResult; + } + } + const summary = yield* withExclusiveFileLock( + input.fs, + codeGraphSnapshotBuildLockPath( + yield* Path.Path, + input.threadnoteHome, + input.identity.checkoutId, + logicalSnapshotId, + ), + { + ...CODE_GRAPH_LOCK_OPTIONS, + onContention: () => + (input.onProgress?.({phase: 'waiting', reason: 'snapshot-build'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + }, + Effect.gen(function* () { + if (!input.force) { + const ready = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, logicalSnapshotId); + if (ready) { + return { + diagnostics: [], + durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, + identity: input.identity, + materialization: {mode: 'reused-snapshot', stagedFiles: 0, totalFiles: cleanInventory.files.length}, + reusedFiles: cleanInventory.files.length - cleanInventory.parsedFiles, + skippedFiles: cleanInventory.skipped, + snapshot: ready, + } satisfies CodeGraphIndexSummary; + } + } + const resumed = input.force + ? yield* input.store.resumableForcedBuild(input.layout.databasePath, logicalSnapshotId) + : undefined; + const building: CodeGraphSnapshot = resumed ?? { + commit: input.identity.headCommit, + dirty: false, + edgeCount: 0, + extractorSet, + fileCount: 0, + graphContentId: graphContentIdentity(extractorSet, cleanInventory.files), + id: snapshotId, + repositoryId: input.identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: input.identity.worktreeId, + }; + const ownerToken = yield* input.store.claimPersistentBuild(input.layout.databasePath, input.identity, building, { + logicalSnapshotId, + owner: input.buildOwner, + }); + return yield* buildAndActivate({ + ...input, + activatePointer: false, + building, + ensureVectors: false, + existing: input.existing, + inventory: cleanInventory, + persistentOwnerToken: ownerToken, + }).pipe( + Effect.catch(cause => + isCodeGraphCapacityPause(cause) + ? Effect.fail(cause) + : input.store + .markFailed(input.layout.databasePath, building.id, messageOf(cause), ownerToken) + .pipe(Effect.andThen(Effect.fail(cause))), + ), + ); + }), + ); + return { + diagnostics: summary.diagnostics, + leaseToken: Option.none(), + snapshot: summary.snapshot, + // Clean builds now materialize directly into a durable `building` + // snapshot. Dirty overlays reuse the ready persisted base instead of a + // connection-private full staging graph. + stagingReusable: false, + } satisfies CommittedBaseResult; +}); + +export const buildAndActivate = Effect.fn('codeGraph.buildAndActivate')(function* (input: { + readonly activatePointer: boolean; + readonly building: CodeGraphSnapshot; + readonly capacityProtection: DirectPersistentCapacityProtection; + readonly committedBase?: CommittedBaseResult; + readonly existing?: CodeGraphSnapshot; + readonly embedding: CodeGraphEmbeddingIndexShape; + readonly ensureVectors: boolean; + readonly force: boolean; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly inventory: CodeGraphInventory; + readonly incrementalAssessment?: IncrementalOverlayAssessment; + readonly incrementalOverlayEnabled?: boolean; + readonly incrementalPrepared?: boolean; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; + readonly persistentOwnerToken?: string; + readonly requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}; + readonly startedAt: number; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; + readonly workspace?: CodeGraphWorkspace; +}) { + const workspace = + input.workspace ?? + input.inventory.workspace ?? + (yield* input.languagePacks.discoverWorkspace(input.inventory.files)); + const directPersistentMaterialization = input.persistentOwnerToken !== undefined; + const protectDirectPersistentWrite = codeGraphDirectPersistentCapacityProtector(input); + const persistentCapacityGuard = protectDirectPersistentWrite; + const extractionDiagnostics: string[] = [...workspace.diagnostics]; + let materializedFiles = 0; + let materializedShardFilesReused = 0; + const reusedFiles = input.inventory.files.length - input.inventory.parsedFiles; + const incrementalAssessment = + input.incrementalAssessment ?? + (input.inventory.dirty ? yield* assessIncrementalOverlay(input, workspace) : undefined); + let fallbackReason: CodeGraphOverlayFallbackReason | undefined = + incrementalAssessment?.mode === 'fallback' + ? incrementalAssessment.reason + : input.existing !== undefined && + input.existing.extractorSet !== input.building.extractorSet && + incrementalAssessment?.mode !== 'eligible' + ? 'extractor-context-changed' + : undefined; + let incrementalApplied = false; + if (incrementalAssessment?.mode === 'eligible') { + const incrementalReusedFiles = input.inventory.files.length - incrementalAssessment.files.length; + if (input.incrementalPrepared !== true) { + yield* input.onProgress?.({ + completed: 0, + phase: 'materializing', + reused: incrementalReusedFiles, + total: incrementalAssessment.files.length, + unit: 'files', + }) ?? Effect.void; + } + incrementalApplied = + input.incrementalPrepared === true + ? true + : incrementalAssessment.reuse === 'persisted-base' + ? yield* input.store.preparePersistedIncrementalActivation( + input.layout.databasePath, + input.committedBase!.snapshot.id, + incrementalAssessment.files, + incrementalAssessment.facts, + { + deletedPaths: incrementalAssessment.deletedPaths, + resolutionClosure: incrementalAssessment.resolutionClosure, + }, + protectDirectPersistentWrite, + ) + : yield* input.store.replaceStagedModifiedFiles( + input.layout.databasePath, + input.committedBase!.snapshot.id, + incrementalAssessment.files, + incrementalAssessment.facts, + protectDirectPersistentWrite, + ); + if (incrementalApplied) { + materializedFiles = incrementalAssessment.files.length; + for (const diagnostic of [ + ...input.committedBase!.diagnostics, + ...incrementalAssessment.facts.flatMap(file => file.diagnostics), + ]) { + if (extractionDiagnostics.length >= 100) break; + if (!extractionDiagnostics.includes(diagnostic)) extractionDiagnostics.push(diagnostic); + } + yield* input.onProgress?.({ + completed: materializedFiles, + phase: 'materializing', + reused: incrementalReusedFiles, + total: incrementalAssessment.files.length, + unit: 'files', + }) ?? Effect.void; + } else { + fallbackReason = 'staging-identity-mismatch'; + } + } + if (!incrementalApplied) { + const attributeFacts = createCachedCodeGraphFactsAttributor(input.inventory.files, workspace); + const shardDerivationIdentity = materializedShardDerivationIdentity( + input.building.extractorSet, + workspace.fingerprint, + graphContentIdentity(input.building.extractorSet, input.inventory.files), + ); + const sourceBytesTotal = input.inventory.files.reduce((total, file) => total + file.size, 0); + const cachedMetadata = yield* cachedFactsMetadata( + input.store, + input.layout.databasePath, + input.inventory.files, + input.languagePacks, + ); + const materializedShards = yield* input.store.loadMaterializedFileShards( + input.layout.databasePath, + input.inventory.files, + input.building.extractorSet, + shardDerivationIdentity, + ); + const materializedShardSetComplete = materializedShards.facts.size === input.inventory.files.length; + if (cachedMetadata.files !== input.inventory.files.length) { + return yield* Effect.fail( + new CodeGraphIndexOperationError( + 'Cached code graph facts are incomplete during materialization planning; retry with a full rebuild.', + ), + ); + } + const batches = factMaterializationBatches(input.inventory.files, cachedMetadata.bytesByPath); + const cachedFactBytesTotal = cachedMetadata.bytes; + const storageEstimate = estimatedMaterializationStorageBytes( + cachedFactBytesTotal, + sourceBytesTotal, + directPersistentMaterialization ? 'direct-persistent' : 'temporary-staged', + 'cached-fact-bytes', + ); + const system = yield* SystemInfo; + const [durableAvailableBytes, temporaryAvailableBytes, durableFilesystem, temporaryFilesystem] = yield* Effect.all( + [ + system.availableDiskBytes(input.layout.repositoryRoot).pipe(Effect.catch(() => Effect.succeed(undefined))), + system.availableDiskBytes(system.tempDirectory).pipe(Effect.catch(() => Effect.succeed(undefined))), + input.fs.stat(input.layout.repositoryRoot).pipe( + Effect.map(info => info.dev), + Effect.option, + ), + input.fs.stat(system.tempDirectory).pipe( + Effect.map(info => info.dev), + Effect.option, + ), + ] as const, + {concurrency: 'unbounded'}, + ); + const filesystemsShared = + Option.isSome(durableFilesystem) && Option.isSome(temporaryFilesystem) + ? durableFilesystem.value === temporaryFilesystem.value + : undefined; + const storagePlan = materializationStoragePlan(storageEstimate, { + durableAvailableBytes, + filesystemsShared, + temporaryAvailableBytes, + }); + let batchesCompleted = 0; + // Final attribution may expand one cached-fact batch into multiple bounded + // write transactions. Until each source batch is decoded, this is a lower + // bound that converges monotonically to the exact finalized receipt count. + let batchesTotal = batches.length; + let sourceBytesCompleted = 0; + let loadingMilliseconds = 0; + let attributionMilliseconds = 0; + let transactionMilliseconds = 0; + let cachedFactBytesCompleted = 0; + let factsBytesCompleted = 0; + let durableDatabaseBytes = 0; + let durableDatabaseHighWaterBytes = 0; + const storageAtStart = yield* materializationStorageFiles(input.fs, input.layout.databasePath); + let durableDatabaseFileBytes = storageAtStart.databaseBytes; + let durableDatabaseFileHighWaterBytes = storageAtStart.databaseBytes; + const durableDatabaseStartBytes = storageAtStart.databaseBytes; + let durableDatabaseGrowthBytes = 0; + let durableDatabaseGrowthHighWaterBytes = 0; + let durableFilesystemBytes = storageAtStart.totalBytes; + let durableFilesystemHighWaterBytes = storageAtStart.totalBytes; + let durableJournalBytes = storageAtStart.journalBytes; + let durableJournalHighWaterBytes = storageAtStart.journalBytes; + let durableSharedMemoryBytes = storageAtStart.sharedMemoryBytes; + let durableSharedMemoryHighWaterBytes = storageAtStart.sharedMemoryBytes; + let durableWalBytes = storageAtStart.walBytes; + let durableWalHighWaterBytes = storageAtStart.walBytes; + let lastStorageFileSampleAt = Number.NEGATIVE_INFINITY; + let temporaryDatabaseBytes = 0; + let temporaryDatabaseHighWaterBytes = 0; + let materializedRows: CodeGraphMaterializationRows = {}; + const stageMilliseconds: Partial> = {}; + const metrics = (finalFactsBytesTotal?: number): CodeGraphMaterializationMetrics => ({ + attributionMilliseconds, + batchesCompleted, + batchesTotal, + cachedFactBytesCompleted, + cachedFactBytesTotal, + ...(fallbackReason === undefined ? {} : {fallbackReason}), + factsBytesCompleted, + ...(finalFactsBytesTotal === undefined ? {} : {factsBytesTotal: finalFactsBytesTotal}), + loadingMilliseconds, + mode: 'full', + rows: materializedRows, + sourceBytesCompleted, + sourceBytesTotal, + stageMilliseconds: {...stageMilliseconds}, + storage: { + ...storagePlan, + durableDatabaseBytes, + durableDatabaseFileBytes, + durableDatabaseFileHighWaterBytes, + durableDatabaseGrowthBytes, + durableDatabaseGrowthHighWaterBytes, + durableDatabaseHighWaterBytes, + durableDatabaseStartBytes, + durableFilesystemBytes, + durableFilesystemHighWaterBytes, + durableJournalBytes, + durableJournalHighWaterBytes, + durableSharedMemoryBytes, + durableSharedMemoryHighWaterBytes, + durableWalBytes, + durableWalHighWaterBytes, + temporaryDatabaseBytes, + temporaryDatabaseHighWaterBytes, + }, + transactionMilliseconds, + }); + const refreshStorageFiles = (force = false) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (!force && now - lastStorageFileSampleAt < 1_000) return; + const current = yield* materializationStorageFiles(input.fs, input.layout.databasePath); + durableDatabaseFileBytes = current.databaseBytes; + durableDatabaseFileHighWaterBytes = Math.max(durableDatabaseFileHighWaterBytes, current.databaseBytes); + durableDatabaseGrowthBytes = Math.max(0, current.databaseBytes - durableDatabaseStartBytes); + durableDatabaseGrowthHighWaterBytes = Math.max(durableDatabaseGrowthHighWaterBytes, durableDatabaseGrowthBytes); + durableFilesystemBytes = current.totalBytes; + durableFilesystemHighWaterBytes = Math.max(durableFilesystemHighWaterBytes, current.totalBytes); + durableJournalBytes = current.journalBytes; + durableJournalHighWaterBytes = Math.max(durableJournalHighWaterBytes, current.journalBytes); + durableSharedMemoryBytes = current.sharedMemoryBytes; + durableSharedMemoryHighWaterBytes = Math.max(durableSharedMemoryHighWaterBytes, current.sharedMemoryBytes); + durableWalBytes = current.walBytes; + durableWalHighWaterBytes = Math.max(durableWalHighWaterBytes, current.walBytes); + lastStorageFileSampleAt = now; + }); + const storageShortfalls = materializationStorageShortfalls(storagePlan); + if (storageShortfalls.length > 0) { + extractionDiagnostics.push( + `Available ${storageShortfalls.join(' and ')} disk space is below the heuristic materialization estimate; ` + + 'indexing will continue while reporting actual TEMP database usage.', + ); + } + yield* input.onProgress?.({ + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + yield* input.store.prepareActivation( + input.layout.databasePath, + input.inventory.files, + directPersistentMaterialization ? input.building.id : undefined, + undefined, + input.persistentOwnerToken, + persistentCapacityGuard, + ); + yield* input.store.stageWorkspaceCatalog(input.layout.databasePath, workspace, persistentCapacityGuard); + let persistentBatchCursor = 0; + const persistentTransactionBatchLimit = input.persistentMaterializationTransactionBatchLimit ?? 4; + interface PendingMaterializationBatch extends PersistentMaterializationTransactionCandidate { + readonly attributionMilliseconds: number; + readonly batchCachedFactBytes: number; + readonly batchFiles: readonly CodeGraphInventoryFile[]; + readonly batchIndex: number; + readonly edges: readonly CodeGraphEdge[]; + readonly loadingMilliseconds: number; + readonly monikers: readonly CodeGraphMonikerV1[]; + readonly references: readonly CodeGraphReference[]; + rows: CodeGraphMaterializationRows; + readonly stageMilliseconds: Map; + readonly symbols: readonly CodeGraphSymbol[]; + } + const pendingBatches: PendingMaterializationBatch[] = []; + const reportStagingProgress = (batch: PendingMaterializationBatch, progress: CodeGraphStagingProgress) => { + if (progress.temporaryDatabaseBytes !== undefined) { + temporaryDatabaseBytes = progress.temporaryDatabaseBytes; + temporaryDatabaseHighWaterBytes = Math.max(temporaryDatabaseHighWaterBytes, progress.temporaryDatabaseBytes); + } + if (progress.durableDatabaseBytes !== undefined) { + durableDatabaseBytes = progress.durableDatabaseBytes; + durableDatabaseHighWaterBytes = Math.max(durableDatabaseHighWaterBytes, progress.durableDatabaseBytes); + } + const activityStage = materializationStagingStage(progress); + const timingKey = progress.stage === 'committed' ? 'committing' : progress.stage; + const previousStageMilliseconds = batch.stageMilliseconds.get(timingKey) ?? 0; + const currentStageMilliseconds = progress.stageElapsedMilliseconds ?? 0; + const stageDeltaMilliseconds = Math.max(0, currentStageMilliseconds - previousStageMilliseconds); + batch.stageMilliseconds.set(timingKey, currentStageMilliseconds); + stageMilliseconds[activityStage] = (stageMilliseconds[activityStage] ?? 0) + stageDeltaMilliseconds; + batch.rows = materializationRowsWithStoreProgress(batch.rows, progress); + return refreshStorageFiles(progress.stage === 'committed').pipe( + Effect.andThen( + input.onProgress?.({ + activity: { + batchCompleted: batch.batchIndex, + batchTotal: batchesTotal, + cachedFactBytes: batch.batchCachedFactBytes, + elapsedMilliseconds: progress.elapsedMilliseconds, + factsBytes: batch.factBytes, + rows: batch.rows, + sourceBytes: batch.sourceBytes, + stage: activityStage, + stageElapsedMilliseconds: currentStageMilliseconds, + transactionMilliseconds: progress.elapsedMilliseconds, + }, + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void, + ), + Effect.catch(() => Effect.void), + ); + }; + const flushPendingBatches = () => + Effect.gen(function* () { + if (pendingBatches.length === 0) return; + const group = pendingBatches.splice(0, pendingBatches.length); + const groupByIndex = new Map(group.map(batch => [batch.batchIndex, batch])); + const transactionStartedAt = yield* Clock.currentTimeMillis; + if (directPersistentMaterialization) { + yield* input.store.stageActivationFactBatches( + input.layout.databasePath, + group.map(batch => ({ + batchIndex: batch.batchIndex, + edges: batch.edges, + finalFactBytes: batch.factBytes, + monikers: batch.monikers, + references: batch.references, + symbols: batch.symbols, + })), + (batchIndex, progress) => reportStagingProgress(groupByIndex.get(batchIndex)!, progress), + persistentCapacityGuard, + ); + } else { + for (const batch of group) { + yield* input.store.stageActivationFacts( + input.layout.databasePath, + batch.symbols, + batch.edges, + batch.references, + progress => reportStagingProgress(batch, progress), + batch.batchIndex, + persistentCapacityGuard, + batch.monikers, + ); + } + } + const groupTransactionMilliseconds = (yield* Clock.currentTimeMillis) - transactionStartedAt; + transactionMilliseconds += groupTransactionMilliseconds; + for (let index = 0; index < group.length; index += 1) { + const batch = group[index]!; + const accountedTransactionMilliseconds = index === group.length - 1 ? groupTransactionMilliseconds : 0; + materializedFiles += batch.fileCount; + batchesCompleted += 1; + sourceBytesCompleted += batch.sourceBytes; + cachedFactBytesCompleted += batch.batchCachedFactBytes; + factsBytesCompleted += batch.factBytes; + materializedRows = addMaterializationRows(materializedRows, batch.rows); + yield* input.onProgress?.({ + activity: { + batchCompleted: batch.batchIndex, + batchTotal: batchesTotal, + cachedFactBytes: batch.batchCachedFactBytes, + elapsedMilliseconds: + batch.loadingMilliseconds + batch.attributionMilliseconds + accountedTransactionMilliseconds, + factsBytes: batch.factBytes, + rows: batch.rows, + sourceBytes: batch.sourceBytes, + stage: 'committing', + transactionMilliseconds: accountedTransactionMilliseconds, + }, + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + } + }); + for (const files of batches) { + const sourceBytes = files.reduce((total, file) => total + file.size, 0); + yield* input.onProgress?.({ + activity: { + batchCompleted: batchesCompleted, + batchTotal: batchesTotal, + sourceBytes, + stage: 'loading-cache', + }, + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + const loadingStartedAt = yield* Clock.currentTimeMillis; + const missingShardFiles = materializedShardSetComplete ? [] : files; + const cached = yield* loadCachedFacts( + input.store, + input.layout.databasePath, + missingShardFiles, + input.languagePacks, + ); + const batchLoadingMilliseconds = (yield* Clock.currentTimeMillis) - loadingStartedAt; + loadingMilliseconds += batchLoadingMilliseconds; + stageMilliseconds['loading-cache'] = loadingMilliseconds; + if (missingShardFiles.some(file => !cached.facts.has(file.path))) { + return yield* Effect.fail( + new CodeGraphIndexOperationError( + 'A cached code graph fact disappeared during indexing; retry with a full rebuild.', + ), + ); + } + yield* input.onProgress?.({ + activity: { + batchCompleted: batchesCompleted, + batchTotal: batchesTotal, + cachedFactBytes: cached.bytes, + elapsedMilliseconds: batchLoadingMilliseconds, + sourceBytes, + stage: 'attributing', + }, + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + const attributionStartedAt = yield* Clock.currentTimeMillis; + const attributedMissingFacts = attributeFacts( + missingShardFiles.map(file => input.languagePacks.postprocessFile(file, cached.facts.get(file.path)!)), + ); + if (missingShardFiles.length > 0) { + yield* input.store.cacheMaterializedFileShards( + input.layout.databasePath, + missingShardFiles, + attributedMissingFacts.map(fact => serializeBoundedCodeGraphFact(fact)), + input.building.extractorSet, + shardDerivationIdentity, + protectDirectPersistentWrite, + ); + } + const attributedMissingByPath = new Map(attributedMissingFacts.map(fact => [fact.path, fact])); + const facts = files.map(file => + materializedShardSetComplete + ? materializedShards.facts.get(file.path)! + : attributedMissingByPath.get(file.path)!, + ); + materializedShardFilesReused += files.length - missingShardFiles.length; + const batchAttributionMilliseconds = (yield* Clock.currentTimeMillis) - attributionStartedAt; + attributionMilliseconds += batchAttributionMilliseconds; + stageMilliseconds.attributing = attributionMilliseconds; + const finalBatches = finalCodeGraphFactBatches(facts); + batchesTotal += Math.max(0, finalBatches.length - 1); + if (extractionDiagnostics.length < 100) { + extractionDiagnostics.push( + ...finalBatches + .flatMap(batch => batch.flatMap(value => value.facts.diagnostics)) + .slice(0, 100 - extractionDiagnostics.length), + ); + } + const filesByPath = new Map(files.map(file => [file.path, file])); + for (let finalBatchIndex = 0; finalBatchIndex < finalBatches.length; finalBatchIndex += 1) { + const finalBatch = finalBatches[finalBatchIndex]!; + const finalFacts = finalBatch.map(value => value.facts); + const batchFinalFactBytes = finalBatch.reduce((total, value) => total + value.bytes, 0); + const batchFiles = finalFacts.map(fact => filesByPath.get(fact.path)!); + const batchSourceBytes = batchFiles.reduce((total, file) => total + file.size, 0); + const batchCachedFactBytes = batchFiles.reduce( + (total, file) => total + (cachedMetadata.bytesByPath.get(file.path) ?? 0), + 0, + ); + const symbols = uniqueById(finalFacts.flatMap(file => file.symbols)); + const relationships = deduplicateMaterializationRelationships( + finalFacts.flatMap(file => file.edges), + finalFacts.flatMap(file => file.references ?? []), + ); + const edges = relationships.edges; + const references = relationships.references; + const monikers = canonicalCodeGraphMonikers(finalFacts.flatMap(file => file.monikers ?? [])); + const rows = materializationRows(symbols, edges.length, references, { + edges: relationships.duplicateEdges, + references: relationships.duplicateReferences, + }); + yield* input.onProgress?.({ + activity: { + batchCompleted: batchesCompleted, + batchTotal: batchesTotal, + cachedFactBytes: batchCachedFactBytes, + elapsedMilliseconds: finalBatchIndex === 0 ? batchAttributionMilliseconds : 0, + factsBytes: batchFinalFactBytes, + rows, + sourceBytes: batchSourceBytes, + stage: 'writing-facts', + }, + completed: materializedFiles, + metrics: metrics(), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + const candidate: PendingMaterializationBatch = { + attributionMilliseconds: finalBatchIndex === 0 ? batchAttributionMilliseconds : 0, + batchCachedFactBytes, + batchFiles, + batchIndex: persistentBatchCursor, + edges, + factBytes: batchFinalFactBytes, + fileCount: batchFiles.length, + loadingMilliseconds: finalBatchIndex === 0 ? batchLoadingMilliseconds : 0, + monikers, + references, + rows, + sourceBytes: batchSourceBytes, + stageMilliseconds: new Map(), + symbols, + }; + if ( + directPersistentMaterialization && + persistentMaterializationTransactionBatches([...pendingBatches, candidate], persistentTransactionBatchLimit) + .length > 1 + ) { + yield* flushPendingBatches(); + } + pendingBatches.push(candidate); + persistentBatchCursor += 1; + const pendingFactsBytes = pendingBatches.reduce((total, batch) => total + batch.factBytes, 0); + const pendingFiles = pendingBatches.reduce((total, batch) => total + batch.fileCount, 0); + const pendingSourceBytes = pendingBatches.reduce((total, batch) => total + batch.sourceBytes, 0); + if ( + !directPersistentMaterialization || + pendingBatches.length >= persistentTransactionBatchLimit || + pendingFiles >= PERSISTENT_MATERIALIZATION_TRANSACTION_FILES || + pendingSourceBytes >= PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES || + pendingFactsBytes >= PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES + ) { + yield* flushPendingBatches(); + } + } + } + yield* flushPendingBatches(); + batchesTotal = persistentBatchCursor; + if (directPersistentMaterialization) { + yield* input.store.finalizePersistentMaterializationPlan( + input.layout.databasePath, + persistentBatchCursor, + persistentCapacityGuard, + ); + } + yield* input.onProgress?.({ + completed: materializedFiles, + metrics: metrics(factsBytesCompleted), + phase: 'materializing', + reused: reusedFiles, + total: input.inventory.files.length, + unit: 'files', + }) ?? Effect.void; + } + const reusableBaseReceipt = input.building.dirty + ? undefined + : { + fileSetFingerprint: reusableBaseFileSetFingerprint(input.inventory.files), + packProvenance: input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), + workspaceFingerprint: workspace.fingerprint, + }; + yield* input.onProgress?.({phase: 'resolving', subphase: 'references'}) ?? Effect.void; + const resolution = yield* input.store.resolveStagedReferences( + input.layout.databasePath, + activity => + ( + input.onProgress?.({ + activity, + phase: 'resolving', + subphase: 'references', + }) ?? Effect.void + ).pipe( + Effect.catch(() => Effect.void), + Effect.andThen(Effect.yieldNow), + ), + persistentCapacityGuard, + ); + const stagedCounts = yield* input.store.stagedFactCounts(input.layout.databasePath); + yield* input.onProgress?.({ + edges: stagedCounts.edges, + phase: 'resolving', + resolved: resolution.resolved, + subphase: 'complete', + symbols: stagedCounts.symbols, + }) ?? Effect.void; + yield* input.store.shrinkMemory(input.layout.databasePath); + + const ready: CodeGraphSnapshot = { + ...input.building, + edgeCount: stagedCounts.edges, + fileCount: input.inventory.files.length, + state: 'ready', + symbolCount: stagedCounts.symbols, + }; + yield* input.onProgress?.({phase: 'activating', snapshotId: ready.id, subphase: 'validating-input'}) ?? Effect.void; + yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); + yield* input.onProgress?.({ + phase: 'activating', + snapshotId: ready.id, + subphase: 'writing-and-checkpointing', + }) ?? Effect.void; + const activatedReady = yield* Effect.gen(function* () { + const activationLease = yield* Effect.acquireRelease( + input.store.activateStaged( + input.layout.databasePath, + input.identity, + ready, + reusableBaseReceipt, + CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS, + activity => + ( + input.onProgress?.({ + activity, + phase: 'activating', + snapshotId: ready.id, + }) ?? Effect.void + ).pipe(Effect.catch(() => Effect.void)), + persistentCapacityGuard, + ), + lease => + Option.match(lease, { + onNone: () => Effect.void, + onSome: token => + input.store.releaseSnapshotLease(input.layout.databasePath, token).pipe(Effect.catch(() => Effect.void)), + }), + ); + const activated = yield* input.store.currentLexicalReadySnapshotById(input.layout.databasePath, ready.id); + if (!activated) { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Activated code graph snapshot could not be read back from its store.'), + ); + } + yield* input.store.shrinkMemory(input.layout.databasePath); + if (input.activatePointer) { + yield* input.onProgress?.({phase: 'activating', snapshotId: activated.id, subphase: 'promoting'}) ?? Effect.void; + // Progress callbacks are user-controlled effects and may yield long enough for + // the worktree to change. Revalidate on both sides of pointer promotion so a + // mutation observed in this window triggers the bounded retry. + yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); + yield* input.store.promote(input.layout.databasePath, input.identity, activated.id, { + persistentCapacityProtector: protectDirectPersistentWrite, + }); + yield* input.store.shrinkMemory(input.layout.databasePath); + yield* verifyIndexInput(input.identity, input.activatePointer, input.threadnoteHome, input.requestedOverlay); + if (Option.isSome(activationLease)) { + yield* input.store.releaseSnapshotLease(input.layout.databasePath, activationLease.value); + } + } + if (input.committedBase && Option.isSome(input.committedBase.leaseToken)) { + yield* input.store.releaseSnapshotLease(input.layout.databasePath, input.committedBase.leaseToken.value); + } + yield* input.onProgress?.({ + phase: 'activating', + snapshotId: activated.id, + subphase: input.activatePointer ? 'structural-ready' : 'complete', + }) ?? Effect.void; + return activated; + }); + let analysisSummaryFailure: string | undefined; + const analysisSummaryBackfilled = + input.activatePointer && !activatedReady.dirty + ? yield* prepareReadyAnalysisSummary({ + databasePath: input.layout.databasePath, + onProgress: input.onProgress, + snapshotId: activatedReady.id, + store: input.store, + }).pipe( + Effect.catch(cause => + Effect.sync(() => { + analysisSummaryFailure = messageOf(cause); + return false; + }), + ), + ) + : yield* ( + input.onProgress?.({ + phase: 'activating', + snapshotId: activatedReady.id, + subphase: 'complete', + }) ?? Effect.void + ).pipe(Effect.as(false)); + const embedding = input.ensureVectors + ? yield* input.embedding + .ensure( + input.threadnoteHome, + input.layout, + activatedReady, + embeddingSymbolSource(input.store, input.layout.databasePath, activatedReady.id), + { + force: input.force, + onProgress: input.onProgress, + }, + ) + .pipe( + Effect.catch(cause => + Effect.succeed({ + embedded: 0, + ready: false, + reason: messageOf(cause), + reused: 0, + } satisfies CodeGraphEmbeddingStatus), + ), + ) + : ({embedded: 0, ready: true, reused: 0} satisfies CodeGraphEmbeddingStatus); + if (input.activatePointer) { + yield* input.fs.remove(input.layout.staleMarkerPath, {force: true}).pipe(Effect.catch(() => Effect.void)); + } + return { + diagnostics: [ + ...(input.inventory.diagnostics ?? []), + ...extractionDiagnostics, + ...(input.inventory.dirty + ? [ + incrementalApplied + ? incrementalAssessment?.mode === 'eligible' && incrementalAssessment.reuse === 'persisted-base' + ? `Dirty overlay reused persisted clean base for ${materializedFiles.toLocaleString()} modified file(s).` + : `Dirty overlay reused clean staging for ${materializedFiles.toLocaleString()} modified file(s).` + : `Dirty overlay used full materialization: ${overlayFallbackDescription(fallbackReason ?? 'staging-unavailable')}.`, + ] + : incrementalApplied + ? [`Clean snapshot reused persisted base for ${materializedFiles.toLocaleString()} modified file(s).`] + : []), + ...(materializedShardFilesReused > 0 + ? [`Reused content-addressed materialized shards for ${materializedShardFilesReused.toLocaleString()} file(s).`] + : []), + ...(analysisSummaryBackfilled ? ['Built the persisted whole-graph analysis summary after promotion.'] : []), + ...(analysisSummaryFailure + ? [`Whole-graph analysis summary will be retried lazily: ${analysisSummaryFailure}`] + : []), + ...(embedding.ready ? [] : [`Vector graph retrieval unavailable: ${embedding.reason ?? 'unknown reason'}`]), + ].slice(0, 100), + durationMs: (yield* Clock.currentTimeMillis) - input.startedAt, + identity: input.identity, + ...(incrementalApplied && incrementalAssessment?.mode === 'eligible' + ? {incrementalWork: incrementalAssessment.work} + : {}), + materialization: { + ...(incrementalApplied && incrementalAssessment?.mode === 'eligible' + ? { + ...(incrementalAssessment.closureProjects === undefined + ? {} + : {closureProjects: incrementalAssessment.closureProjects}), + ...(incrementalAssessment.resolutionClosure === undefined + ? {} + : {resolutionClosure: incrementalAssessment.resolutionClosure}), + } + : {}), + ...(fallbackReason ? {fallbackReason} : {}), + mode: incrementalApplied ? (input.inventory.dirty ? 'incremental-overlay' : 'incremental-clean') : 'full', + stagedFiles: materializedFiles, + totalFiles: input.inventory.files.length, + }, + reusedFiles: input.inventory.files.length - input.inventory.parsedFiles, + skippedFiles: input.inventory.skipped, + snapshot: activatedReady, + } satisfies CodeGraphIndexSummary; +}); diff --git a/src/code_graph/indexer_incremental.ts b/src/code_graph/indexer_incremental.ts new file mode 100644 index 00000000..279d6d7c --- /dev/null +++ b/src/code_graph/indexer_incremental.ts @@ -0,0 +1,875 @@ +import {Effect} from 'effect'; +import {sha256HexSync} from '../crypto/sha256.js'; +import {createRepositoryFactAttributor} from './extractor.js'; +import {finalCodeGraphFactBatches, serializeBoundedCodeGraphFact} from './fact_budget.js'; +import { + assessProjectClosureSeeds, + planProjectIncrementalClosure, + PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES, + PROJECT_INCREMENTAL_CLOSURE_MAX_FILES, + PROJECT_INCREMENTAL_CLOSURE_MAX_SOURCE_BYTES, + selectProjectIncrementalClosure, +} from './incremental_closure.js'; +import {codeGraphIncrementalWorkFitsBudget, measureCodeGraphIncrementalWork} from './incremental_work.js'; +import { + cachedFactsMetadata, + extractorSetIdentity, + extractorSetIdentityFromPackProvenance, + graphContentIdentity, + loadCachedFacts, + loadCachedFactsWithPackProvenance, +} from './indexer_materialization.js'; +import {inventoryFilesForPaths} from './indexer_shared.js'; +import type { + CommittedBaseResult, + IncrementalOverlayAssessment, + IncrementalOverlayPreassessment, +} from './indexer_types.js'; +import type {CodeGraphInventory} from './inventory.js'; +import {assessCodeGraphLanguagePackDelta} from './languages/provenance.js'; +import type {CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; +import type {CodeGraphWorkspace} from './languages/types.js'; +import type {CodeGraphLayout} from './layout.js'; +import {compareCodeUnits} from './ordering.js'; +import { + CODE_GRAPH_REUSABLE_BASE_RECEIPT_VERSION, + materializedShardDerivationIdentity, + type CodeGraphDirectPersistentCapacityProtector, + type CodeGraphReusableCleanBase, + type CodeGraphReusableReexport, + type CodeGraphReusableReexportSeed, + type CodeGraphStoreShape, +} from './store.js'; +import { + type CodeGraphEdge, + type CodeGraphFileFacts, + type CodeGraphInventoryFile, + type CodeGraphOverlayFallbackReason, + type CodeGraphReference, + type CodeGraphSnapshot, + type CodeGraphSymbol, +} from './types.js'; +import {createWorkspaceAttributor} from './workspace.js'; +import {assessCodeGraphWorkspaceCompatibility} from './workspace_compatibility.js'; + +export const assessIncrementalOverlay = Effect.fn('codeGraph.assessIncrementalOverlay')(function* ( + input: { + readonly building: CodeGraphSnapshot; + readonly committedBase?: CommittedBaseResult; + readonly force: boolean; + readonly incrementalOverlayEnabled?: boolean; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly store: CodeGraphStoreShape; + }, + workspace: CodeGraphWorkspace, + suppliedPreassessment?: IncrementalOverlayPreassessment, +) { + if (input.incrementalOverlayEnabled === false) { + return {mode: 'fallback', reason: 'disabled'} satisfies IncrementalOverlayAssessment; + } + if (input.force) return {mode: 'fallback', reason: 'forced-full-rebuild'} satisfies IncrementalOverlayAssessment; + const preassessment: IncrementalOverlayPreassessment = + suppliedPreassessment ?? + (yield* assessIncrementalOverlayCompatibility( + { + extractorSet: input.building.extractorSet, + inventory: input.inventory, + languagePacks: input.languagePacks, + layout: input.layout, + store: input.store, + }, + workspace, + )); + if (preassessment.mode === 'fallback') return preassessment; + if (!input.committedBase) + return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; + if ( + input.building.extractorSet !== input.committedBase.snapshot.extractorSet && + preassessment.extractorTransition !== true + ) { + return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayAssessment; + } + let reuse: 'persisted-base' | 'staged-base' = 'staged-base'; + if (!input.committedBase.stagingReusable) { + const receipt = yield* input.store.reusableBaseReceipt(input.layout.databasePath, input.committedBase.snapshot.id); + if ( + !receipt || + receipt.formatVersion !== CODE_GRAPH_REUSABLE_BASE_RECEIPT_VERSION || + receipt.resolutionSurfaceVersion !== 1 || + receipt.workspaceFingerprint !== preassessment.committedWorkspace.fingerprint || + (preassessment.resolutionClosure !== 'full' && + receipt.fileSetFingerprint !== reusableBaseFileSetFingerprint(input.inventory.committedFiles)) + ) { + return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; + } + reuse = 'persisted-base'; + } + let reusableFacts = preassessment.facts; + if (reuse === 'persisted-base' && preassessment.resolutionClosure !== 'full') { + const affectedPaths = + preassessment.resolutionClosure === 'project' ? new Set(preassessment.files.map(file => file.path)) : undefined; + const seeds = reusableReexportSeeds(preassessment.facts).filter(seed => !affectedPaths?.has(seed.path)); + if (seeds.length > 0) { + const reexports = yield* input.store.reusableReexports( + input.layout.databasePath, + input.committedBase.snapshot.id, + seeds, + {maxRows: 10_000}, + ); + if (reexports === undefined) { + return {mode: 'fallback', reason: 'staging-unavailable'} satisfies IncrementalOverlayAssessment; + } + if (reexports.length > 10_000) { + return {mode: 'fallback', reason: 'reexport-closure-unbounded'} satisfies IncrementalOverlayAssessment; + } + if (preassessment.resolutionClosure === 'project') { + if ( + reexports.some(reexport => affectedPaths!.has(reexport.sourcePath) || affectedPaths!.has(reexport.targetPath)) + ) { + return {mode: 'fallback', reason: 'project-closure-incomplete'} satisfies IncrementalOverlayAssessment; + } + } + const enrichedFacts = enrichPersistedTypeScriptReexports(preassessment.facts, reexports); + if (!enrichedFacts) { + return {mode: 'fallback', reason: 'reexport-closure-unbounded'} satisfies IncrementalOverlayAssessment; + } + reusableFacts = enrichedFacts; + } + } + const finalBatches = finalCodeGraphFactBatches(reusableFacts); + if (finalBatches.length !== 1 && preassessment.resolutionClosure !== 'full') { + return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayAssessment; + } + const facts = finalBatches.flatMap(batch => batch.map(value => value.facts)); + const work = measureCodeGraphIncrementalWork({ + deletedPaths: preassessment.deletedPaths, + facts, + files: preassessment.files, + totalFiles: input.inventory.files.length, + }); + if (!codeGraphIncrementalWorkFitsBudget(work)) { + return {mode: 'fallback', reason: 'incremental-rewrite-unbounded'} satisfies IncrementalOverlayAssessment; + } + return { + closureProjects: preassessment.closureProjects, + deletedPaths: preassessment.deletedPaths, + facts, + files: preassessment.files, + mode: 'eligible', + resolutionClosure: preassessment.resolutionClosure, + reuse, + work, + } satisfies IncrementalOverlayAssessment; +}); + +export const assessIncrementalOverlayCompatibility = Effect.fn('codeGraph.assessIncrementalOverlayCompatibility')( + function* ( + input: { + readonly extractorSet: string; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly store: CodeGraphStoreShape; + }, + workspace: CodeGraphWorkspace, + ) { + if (input.extractorSet !== extractorSetIdentity(input.inventory.committedFiles, input.languagePacks)) { + return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayPreassessment; + } + const committedWorkspace = + input.inventory.workspace ?? (yield* input.languagePacks.discoverWorkspace(input.inventory.committedFiles)); + const committedByPath = new Map(input.inventory.committedFiles.map(file => [file.path, file])); + const effectiveByPath = new Map(input.inventory.files.map(file => [file.path, file])); + if ( + committedByPath.size !== effectiveByPath.size || + [...committedByPath].some(([path]) => !effectiveByPath.has(path)) + ) { + return {mode: 'fallback', reason: 'file-set-changed'} satisfies IncrementalOverlayPreassessment; + } + const modifiedFiles = input.inventory.files.filter(file => { + const committed = committedByPath.get(file.path)!; + return ( + committed.contentHash !== file.contentHash || + committed.language !== file.language || + committed.mode !== file.mode || + committed.size !== file.size || + committed.source !== file.source + ); + }); + if (modifiedFiles.length === 0) { + return {mode: 'fallback', reason: 'no-materialized-changes'} satisfies IncrementalOverlayPreassessment; + } + const workspaceCompatibility = assessCodeGraphWorkspaceCompatibility(committedWorkspace, workspace); + if (workspaceCompatibility.mode === 'fallback') { + return workspaceCompatibility satisfies IncrementalOverlayPreassessment; + } + const committedFiles = modifiedFiles.map(file => committedByPath.get(file.path)!); + const changedDecodeBudget = yield* assessProjectClosureChangedDecodeBudget({ + baseFiles: committedFiles, + currentFiles: modifiedFiles, + databasePath: input.layout.databasePath, + languagePacks: input.languagePacks, + store: input.store, + }); + if (changedDecodeBudget.mode === 'fallback') return changedDecodeBudget; + const [committedCache, effectiveCache] = yield* Effect.all( + [ + loadCachedFacts(input.store, input.layout.databasePath, committedFiles, input.languagePacks), + loadCachedFacts(input.store, input.layout.databasePath, modifiedFiles, input.languagePacks), + ], + {concurrency: 1}, + ); + if ( + committedFiles.some(file => !committedCache.facts.has(file.path)) || + modifiedFiles.some(file => !effectiveCache.facts.has(file.path)) + ) { + return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; + } + const committedRawFacts = committedFiles.map(file => + input.languagePacks.postprocessFile(file, committedCache.facts.get(file.path)!), + ); + const effectiveRawFacts = modifiedFiles.map(file => + input.languagePacks.postprocessFile(file, effectiveCache.facts.get(file.path)!), + ); + const committedFacts = attributeInventoryFacts( + input.inventory.committedFiles, + committedWorkspace, + committedRawFacts, + ); + const effectiveFacts = attributeInventoryFacts(input.inventory.files, workspace, effectiveRawFacts); + const committedFactsByPath = new Map(committedFacts.map(file => [file.path, file])); + const resolutionSurfaceChanged = effectiveFacts.some(file => { + const committed = committedFactsByPath.get(file.path); + return !committed || !hasSameCodeGraphResolutionSurface(committed.symbols, file.symbols); + }); + const dynamicAliases = hasDynamicAliases(committedFacts) || hasDynamicAliases(effectiveFacts); + if (!dynamicAliases && !resolutionSurfaceChanged && workspaceCompatibility.mode === 'unchanged') { + if (finalCodeGraphFactBatches(effectiveFacts).length !== 1) { + return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; + } + return { + committedWorkspace, + facts: effectiveFacts, + files: modifiedFiles, + mode: 'compatible', + } satisfies IncrementalOverlayPreassessment; + } + return yield* assessProjectIncrementalClosureCompatibility({ + baseWorkspace: committedWorkspace, + changedBaseFacts: committedFacts, + changedCurrentFacts: effectiveFacts, + currentChangedFiles: modifiedFiles, + currentFiles: input.inventory.files, + currentWorkspace: workspace, + languagePacks: input.languagePacks, + layout: input.layout, + store: input.store, + workspaceSeedProjectIds: + workspaceCompatibility.mode === 'project-closure' ? workspaceCompatibility.seedProjectIds : [], + }); + }, +); + +export const assessReusableCleanBaseCompatibility = Effect.fn('codeGraph.assessReusableCleanBaseCompatibility')( + function* ( + input: { + readonly candidate: CodeGraphReusableCleanBase; + readonly inventory: CodeGraphInventory; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; + readonly store: CodeGraphStoreShape; + }, + workspace: CodeGraphWorkspace, + modifiedFiles: readonly CodeGraphInventoryFile[], + ) { + const currentExtractorSet = extractorSetIdentity(input.inventory.files, input.languagePacks); + const extractorTransition = input.candidate.snapshot.extractorSet !== currentExtractorSet; + const packDelta = extractorTransition + ? assessCodeGraphLanguagePackDelta( + input.candidate.receipt.packProvenance, + input.languagePacks.activePackProvenance(input.inventory.files.map(file => file.path)), + ) + : ({changedPackIds: [], mode: 'compatible'} as const); + if ( + packDelta.mode === 'fallback' || + (extractorTransition && + input.candidate.snapshot.extractorSet !== + extractorSetIdentityFromPackProvenance(input.candidate.receipt.packProvenance)) + ) { + return {mode: 'fallback', reason: 'extractor-context-changed'} satisfies IncrementalOverlayPreassessment; + } + if (input.candidate.receipt.workspaceFingerprint !== workspace.fingerprint) { + return {mode: 'fallback', reason: 'workspace-changed'} satisfies IncrementalOverlayPreassessment; + } + if (modifiedFiles.length === 0) { + return {mode: 'fallback', reason: 'no-materialized-changes'} satisfies IncrementalOverlayPreassessment; + } + const baseFiles = inventoryFilesForPaths( + input.candidate.files, + modifiedFiles.map(file => file.path), + ); + if (!baseFiles) { + return {mode: 'fallback', reason: 'file-set-changed'} satisfies IncrementalOverlayPreassessment; + } + const changedDecodeBudget = extractorTransition + ? projectClosureSourceBudgetFits(baseFiles) && projectClosureSourceBudgetFits(modifiedFiles) + ? ({mode: 'eligible'} as const) + : ({mode: 'fallback', reason: 'project-closure-unbounded'} as const) + : yield* assessProjectClosureChangedDecodeBudget({ + baseFiles, + currentFiles: modifiedFiles, + databasePath: input.layout.databasePath, + languagePacks: input.languagePacks, + store: input.store, + }); + if (changedDecodeBudget.mode === 'fallback') return changedDecodeBudget; + const [baseCache, currentCache] = yield* Effect.all( + [ + extractorTransition + ? loadCachedFactsWithPackProvenance( + input.store, + input.layout.databasePath, + baseFiles, + input.languagePacks, + input.candidate.receipt.packProvenance, + ) + : loadCachedFacts(input.store, input.layout.databasePath, baseFiles, input.languagePacks), + loadCachedFacts(input.store, input.layout.databasePath, modifiedFiles, input.languagePacks), + ], + {concurrency: 1}, + ); + if ( + baseFiles.some(file => !baseCache.facts.has(file.path)) || + modifiedFiles.some(file => !currentCache.facts.has(file.path)) + ) { + return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; + } + if ( + extractorTransition && + (baseCache.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES || + currentCache.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES) + ) { + return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; + } + const baseRawFacts = baseFiles.map(file => + input.languagePacks.postprocessFile(file, baseCache.facts.get(file.path)!), + ); + const currentRawFacts = modifiedFiles.map(file => + input.languagePacks.postprocessFile(file, currentCache.facts.get(file.path)!), + ); + const baseFacts = attributeInventoryFacts(input.candidate.files, workspace, baseRawFacts); + const currentFacts = attributeInventoryFacts(input.inventory.files, workspace, currentRawFacts); + const baseFactsByPath = new Map(baseFacts.map(file => [file.path, file])); + const resolutionSurfaceChanged = currentFacts.some(file => { + const base = baseFactsByPath.get(file.path); + return !base || !hasSameCodeGraphResolutionSurface(base.symbols, file.symbols); + }); + const dynamicAliases = hasDynamicAliases(baseFacts) || hasDynamicAliases(currentFacts); + if (dynamicAliases || resolutionSurfaceChanged) { + const closure = yield* assessProjectIncrementalClosureCompatibility({ + baseWorkspace: workspace, + changedBaseFacts: baseFacts, + changedCurrentFacts: currentFacts, + currentChangedFiles: modifiedFiles, + currentFiles: input.inventory.files, + currentWorkspace: workspace, + languagePacks: input.languagePacks, + layout: input.layout, + store: input.store, + workspaceSeedProjectIds: [], + }); + return closure.mode === 'compatible' && extractorTransition + ? {...closure, extractorTransition: true as const} + : closure; + } + if (finalCodeGraphFactBatches(currentFacts).length !== 1) { + return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; + } + yield* input.store.cacheMaterializedFileShards( + input.layout.databasePath, + modifiedFiles, + currentFacts.map(fact => serializeBoundedCodeGraphFact(fact)), + currentExtractorSet, + materializedShardDerivationIdentity( + currentExtractorSet, + workspace.fingerprint, + graphContentIdentity(currentExtractorSet, input.inventory.files), + ), + input.persistentCapacityProtector, + ); + return { + committedWorkspace: workspace, + ...(extractorTransition ? {extractorTransition: true as const} : {}), + facts: currentFacts, + files: modifiedFiles, + mode: 'compatible', + } satisfies IncrementalOverlayPreassessment; + }, +); + +const assessProjectIncrementalClosureCompatibility = Effect.fn( + 'codeGraph.assessProjectIncrementalClosureCompatibility', +)(function* (input: { + readonly baseWorkspace: CodeGraphWorkspace; + readonly changedBaseFacts: readonly CodeGraphFileFacts[]; + readonly changedCurrentFacts: readonly CodeGraphFileFacts[]; + readonly currentChangedFiles: readonly CodeGraphInventoryFile[]; + readonly currentFiles: readonly CodeGraphInventoryFile[]; + readonly currentWorkspace: CodeGraphWorkspace; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly layout: CodeGraphLayout; + readonly store: CodeGraphStoreShape; + readonly workspaceSeedProjectIds: readonly string[]; +}) { + const seeds = assessProjectClosureSeeds({ + committedFacts: input.changedBaseFacts, + effectiveFacts: input.changedCurrentFacts, + projects: input.currentWorkspace.projects, + }); + if (seeds.mode === 'fallback') { + return seeds satisfies IncrementalOverlayPreassessment; + } + const seedProjectIds = [...new Set([...seeds.seedProjectIds, ...input.workspaceSeedProjectIds])].sort( + compareCodeUnits, + ); + const selection = selectProjectIncrementalClosure({ + files: input.currentFiles, + modifiedPaths: input.currentChangedFiles.map(file => file.path), + projects: input.currentWorkspace.projects, + seedProjectIds, + workspaceDiagnostics: input.currentWorkspace.diagnostics, + }); + if (selection.mode === 'fallback') { + return selection satisfies IncrementalOverlayPreassessment; + } + const currentByPath = new Map(input.currentFiles.map(file => [file.path, file])); + const affectedFiles = selection.affectedPaths.map(path => currentByPath.get(path)!); + const metadata = yield* cachedFactsMetadata( + input.store, + input.layout.databasePath, + affectedFiles, + input.languagePacks, + ); + const plan = planProjectIncrementalClosure({ + cachedFactBytesByPath: metadata.bytesByPath, + files: input.currentFiles, + modifiedPaths: input.currentChangedFiles.map(file => file.path), + projects: input.currentWorkspace.projects, + seedProjectIds, + workspaceDiagnostics: input.currentWorkspace.diagnostics, + }); + if (plan.mode === 'fallback') { + return plan satisfies IncrementalOverlayPreassessment; + } + if (metadata.files !== affectedFiles.length || plan.affectedPaths.length !== affectedFiles.length) { + return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; + } + const currentCache = yield* loadCachedFacts( + input.store, + input.layout.databasePath, + affectedFiles, + input.languagePacks, + ); + if (affectedFiles.some(file => !currentCache.facts.has(file.path))) { + return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; + } + const currentRawFacts = affectedFiles.map(file => + input.languagePacks.postprocessFile(file, currentCache.facts.get(file.path)!), + ); + const currentFacts = attributeInventoryFacts(input.currentFiles, input.currentWorkspace, currentRawFacts); + if (finalCodeGraphFactBatches(currentFacts).length !== 1) { + return {mode: 'fallback', reason: 'fact-budget-expanded'} satisfies IncrementalOverlayPreassessment; + } + return { + closureProjects: plan.projectIds.length, + committedWorkspace: input.baseWorkspace, + facts: currentFacts, + files: affectedFiles, + mode: 'compatible', + resolutionClosure: 'project', + } satisfies IncrementalOverlayPreassessment; +}); + +const assessProjectClosureChangedDecodeBudget = Effect.fn('codeGraph.assessProjectClosureChangedDecodeBudget')( + function* (input: { + readonly baseFiles: readonly CodeGraphInventoryFile[]; + readonly currentFiles: readonly CodeGraphInventoryFile[]; + readonly databasePath: string; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly store: CodeGraphStoreShape; + }) { + if (!projectClosureSourceBudgetFits(input.baseFiles) || !projectClosureSourceBudgetFits(input.currentFiles)) { + return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; + } + const [baseMetadata, currentMetadata] = yield* Effect.all( + [ + cachedFactsMetadata(input.store, input.databasePath, input.baseFiles, input.languagePacks), + cachedFactsMetadata(input.store, input.databasePath, input.currentFiles, input.languagePacks), + ], + {concurrency: 1}, + ); + if (baseMetadata.files !== input.baseFiles.length || currentMetadata.files !== input.currentFiles.length) { + return {mode: 'fallback', reason: 'cache-incomplete'} satisfies IncrementalOverlayPreassessment; + } + if ( + baseMetadata.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES || + currentMetadata.bytes > PROJECT_INCREMENTAL_CLOSURE_MAX_CACHED_FACT_BYTES + ) { + return {mode: 'fallback', reason: 'project-closure-unbounded'} satisfies IncrementalOverlayPreassessment; + } + return {mode: 'eligible'} as const; + }, +); + +function projectClosureSourceBudgetFits(files: readonly CodeGraphInventoryFile[]): boolean { + if (files.length > PROJECT_INCREMENTAL_CLOSURE_MAX_FILES) return false; + let sourceBytes = 0; + for (const file of files) { + if (!Number.isSafeInteger(file.size) || file.size < 0) return false; + if (file.size > PROJECT_INCREMENTAL_CLOSURE_MAX_SOURCE_BYTES - sourceBytes) return false; + sourceBytes += file.size; + } + return true; +} + +function reusableReexportSeeds(facts: readonly CodeGraphFileFacts[]): readonly CodeGraphReusableReexportSeed[] { + const seeds = facts.flatMap(file => + (file.references ?? []).flatMap(reference => + reference.resolutionDomain === 'typescript' && isPersistedReexportEnrichableRelation(reference.relation) + ? reference.lookupTiers.flatMap(tier => tier.flatMap(parseTypeScriptPathNameLookupKey)) + : [], + ), + ); + return uniqueByKey(seeds, seed => `${seed.path}\0${seed.name}`); +} + +function enrichPersistedTypeScriptReexports( + facts: readonly CodeGraphFileFacts[], + reexports: readonly CodeGraphReusableReexport[], +): readonly CodeGraphFileFacts[] | undefined { + if (reexports.length === 0) return facts; + const provenance = new Map(); + for (const reexport of reexports) { + const key = `${reexport.sourcePath}\0${reexport.localName}`; + const values = provenance.get(key) ?? []; + values.push(reexport); + provenance.set(key, values); + } + const terminalResolver = createPersistedReexportTerminalResolver(provenance); + const enriched = facts.map(file => { + if (!file.references) return file; + return { + ...file, + references: file.references.map(reference => + enrichPersistedTypeScriptReference(reference, provenance, terminalResolver), + ), + }; + }); + return terminalResolver.exhausted() ? undefined : enriched; +} + +function enrichPersistedTypeScriptReference( + reference: CodeGraphReference, + provenance: ReadonlyMap, + terminalResolver: PersistedReexportTerminalResolver, +): CodeGraphReference { + if (reference.resolutionDomain !== 'typescript' || !isPersistedReexportEnrichableRelation(reference.relation)) { + return reference; + } + const parsedTargets = reference.lookupTiers.flatMap(tier => tier.flatMap(parseTypeScriptPathNameLookupTarget)); + if (!parsedTargets.some(target => provenance.has(`${target.path}\0${target.name}`))) return reference; + return { + ...reference, + lookupTiers: reference.lookupTiers + .map(tier => + uniqueStrings( + tier.flatMap(key => { + const parsed = parseTypeScriptPathNameLookupTarget(key); + if (parsed.length === 0) return [key]; + return parsed.flatMap(target => + (terminalResolver.resolve(target) ?? []).map( + terminal => + `${target.lookupPrefix}path:${encodeURIComponent(terminal.path)}:name:${encodeURIComponent(terminal.name)}${target.lookupSuffix}`, + ), + ); + }), + ), + ) + .filter(tier => tier.length > 0), + }; +} + +function isPersistedReexportEnrichableRelation(relation: CodeGraphEdge['relation']): boolean { + return ['calls', 'constructs', 'exports', 'extends', 'implements', 'overrides', 'references'].includes(relation); +} + +const PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS = 40_000; +const PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS = 10_000; + +type PersistedReexportTerminalTraversal = + | { + readonly mode: 'complete'; + readonly operations: number; + readonly targets: readonly CodeGraphReusableReexportSeed[]; + } + | { + readonly mode: 'fallback'; + readonly reason: 'reexport-closure-unbounded'; + }; + +interface PersistedReexportTerminalResolver { + readonly exhausted: () => boolean; + readonly resolve: (target: CodeGraphReusableReexportSeed) => readonly CodeGraphReusableReexportSeed[] | undefined; +} + +function createPersistedReexportTerminalResolver( + provenance: ReadonlyMap, +): PersistedReexportTerminalResolver { + const cache = new Map(); + let operationsRemaining = PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS; + let traversalExhausted = false; + return { + exhausted: () => traversalExhausted, + resolve: target => { + const key = reusableReexportSeedKey(target); + const cached = cache.get(key); + if (cached) return cached; + if (traversalExhausted) return undefined; + const traversal = resolvePersistedReexportTerminals(target, provenance, { + maxOperations: operationsRemaining, + maxTerminals: PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS, + }); + if (traversal.mode === 'fallback') { + traversalExhausted = true; + return undefined; + } + operationsRemaining -= traversal.operations; + cache.set(key, traversal.targets); + return traversal.targets; + }, + }; +} + +export function resolvePersistedReexportTerminals( + target: CodeGraphReusableReexportSeed, + provenance: ReadonlyMap, + options: {readonly maxOperations?: number; readonly maxTerminals?: number} = {}, +): PersistedReexportTerminalTraversal { + const maxOperations = options.maxOperations ?? PERSISTED_REEXPORT_ENRICHMENT_MAX_OPERATIONS; + const maxTerminals = options.maxTerminals ?? PERSISTED_REEXPORT_ENRICHMENT_MAX_TERMINALS; + if ( + !Number.isSafeInteger(maxOperations) || + maxOperations < 0 || + !Number.isSafeInteger(maxTerminals) || + maxTerminals < 0 + ) { + return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; + } + const discovered = new Set([reusableReexportSeedKey(target)]); + const pending = [target]; + const terminals = new Map(); + let operations = 0; + const consumeOperation = (): boolean => { + if (operations >= maxOperations) return false; + operations += 1; + return true; + }; + while (pending.length > 0) { + if (!consumeOperation()) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; + const current = pending.pop()!; + const next = [...(provenance.get(reusableReexportSeedKey(current)) ?? [])].sort((left, right) => + compareCodeUnits(reusableReexportKey(left), reusableReexportKey(right)), + ); + if (next.length === 0) { + terminals.set(reusableReexportSeedKey(current), current); + if (terminals.size > maxTerminals) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; + continue; + } + for (let index = next.length - 1; index >= 0; index -= 1) { + if (!consumeOperation()) return {mode: 'fallback', reason: 'reexport-closure-unbounded'}; + const reexport = next[index]!; + const candidate = {name: reexport.importedName, path: reexport.targetPath}; + const key = reusableReexportSeedKey(candidate); + if (discovered.has(key)) continue; + discovered.add(key); + pending.push(candidate); + } + } + if (terminals.size === 0) terminals.set(reusableReexportSeedKey(target), target); + return { + mode: 'complete', + operations, + targets: [...terminals.values()].sort((left, right) => + compareCodeUnits(reusableReexportSeedKey(left), reusableReexportSeedKey(right)), + ), + }; +} + +function reusableReexportSeedKey(value: CodeGraphReusableReexportSeed): string { + return `${value.path}\0${value.name}`; +} + +function reusableReexportKey(value: CodeGraphReusableReexport): string { + return `${value.sourcePath}\0${value.localName}\0${value.targetPath}\0${value.importedName}`; +} + +function parseTypeScriptPathNameLookupKey(value: string): readonly CodeGraphReusableReexportSeed[] { + return parseTypeScriptPathNameLookupTarget(value).map(({name, path}) => ({name, path})); +} + +interface TypeScriptPathNameLookupTarget extends CodeGraphReusableReexportSeed { + readonly lookupPrefix: string; + readonly lookupSuffix: string; +} + +function parseTypeScriptPathNameLookupTarget(value: string): readonly TypeScriptPathNameLookupTarget[] { + const match = + /^typescript:((?:[^:]+:)?)path:([^:]+):name:([^:]+)(:(?:arity:\d+|implementation|merge-canonical))?$/.exec(value); + if (!match) return []; + try { + return [ + { + lookupPrefix: `typescript:${match[1]!}`, + lookupSuffix: match[4] ?? '', + name: decodeURIComponent(match[3]!), + path: decodeURIComponent(match[2]!), + }, + ]; + } catch { + return []; + } +} + +function uniqueByKey(values: readonly A[], keyOf: (value: A) => string): readonly A[] { + const output = new Map(); + for (const value of values) { + const key = keyOf(value); + if (!output.has(key)) output.set(key, value); + } + return [...output.values()]; +} + +function uniqueStrings(values: readonly string[]): readonly string[] { + return [...new Set(values)]; +} + +export function reusableBaseFileSetFingerprint(files: readonly CodeGraphInventoryFile[]): string { + return sha256HexSync( + `reusable-base-file-set-v1\n${files + .map(file => `${file.path}\0${file.language}\0${file.mode}`) + .sort(compareCodeUnits) + .join('\n')}`, + ); +} + +function attributeInventoryFacts( + files: readonly CodeGraphInventoryFile[], + workspace: CodeGraphWorkspace, + facts: readonly CodeGraphFileFacts[], +): readonly CodeGraphFileFacts[] { + return deriveCachedCodeGraphFacts(files, workspace, facts); +} + +/** + * Rehydrates parser-only cached facts into the current repository derivation. + * Resolution must precede workspace scoping because raw parser references can + * intentionally defer their lookup tiers until the whole file set is known. + */ +export function deriveCachedCodeGraphFacts( + files: readonly CodeGraphInventoryFile[], + workspace: CodeGraphWorkspace, + facts: readonly CodeGraphFileFacts[], +): readonly CodeGraphFileFacts[] { + return createCachedCodeGraphFactsAttributor(files, workspace)(facts); +} + +export function createCachedCodeGraphFactsAttributor( + files: readonly CodeGraphInventoryFile[], + workspace: CodeGraphWorkspace, +): (facts: readonly CodeGraphFileFacts[]) => readonly CodeGraphFileFacts[] { + const attributeRepositoryFacts = createRepositoryFactAttributor(files); + const attributeWorkspace = createWorkspaceAttributor(workspace); + return facts => attributeWorkspace(attributeRepositoryFacts(facts)); +} + +function hasDynamicAliases(facts: readonly CodeGraphFileFacts[]): boolean { + return facts.some(file => file.references?.some(reference => (reference.aliasLookupKeys?.length ?? 0) > 0) === true); +} + +export function hasSameCodeGraphResolutionSurface( + left: readonly CodeGraphSymbol[], + right: readonly CodeGraphSymbol[], +): boolean { + if (left.length !== right.length) return false; + const leftById = new Map(); + for (const symbol of left) { + if (leftById.has(symbol.id)) return false; + leftById.set(symbol.id, symbolResolutionSurface(symbol)); + } + const rightIds = new Set(); + for (const symbol of right) { + if (rightIds.has(symbol.id)) return false; + rightIds.add(symbol.id); + if (leftById.get(symbol.id) !== symbolResolutionSurface(symbol)) return false; + } + return true; +} + +function symbolResolutionSurface(symbol: CodeGraphSymbol): string { + // Signature, content, documentation, and spans are replaced with the changed file's facts but do not affect + // cross-file endpoint resolution. The current resolver's complete lookup contract is serialized below. + return JSON.stringify({ + arity: symbol.arity, + exported: symbol.exported, + id: symbol.id, + kind: symbol.kind, + language: symbol.language, + lookupKeys: symbol.lookupKeys ?? [], + name: symbol.name, + packageName: symbol.packageName, + path: symbol.path, + qualifiedName: symbol.qualifiedName, + resolutionDomain: symbol.resolutionDomain, + resolutionScopeId: symbol.resolutionScopeId, + }); +} + +export function overlayFallbackDescription(reason: CodeGraphOverlayFallbackReason): string { + switch (reason) { + case 'cache-incomplete': + return 'cached facts were incomplete'; + case 'disabled': + return 'incremental overlay reuse was disabled'; + case 'dynamic-aliases': + return 'changed files participate in dynamic alias resolution'; + case 'extractor-context-changed': + return 'resolution context changed'; + case 'fact-budget-expanded': + return 'final attributed facts exceeded one bounded incremental transaction'; + case 'file-set-changed': + return 'eligible files were added or deleted'; + case 'forced-full-rebuild': + return 'a full rebuild was requested'; + case 'incremental-rewrite-unbounded': + return 'the changed closure exceeded the bounded incremental rewrite budget'; + case 'no-materialized-changes': + return 'no graph-eligible file content changed'; + case 'project-closure-incomplete': + return 'the declared project dependency closure was incomplete or ambiguous'; + case 'project-closure-unbounded': + return 'the project dependency closure exceeded one bounded materialization batch'; + case 'reexport-closure-unbounded': + return 'persisted reexport provenance exceeded the bounded project-closure lookup'; + case 'resolution-surface-changed': + return 'a declaration or lookup surface changed'; + case 'staging-identity-mismatch': + return 'the reusable staging identity was not current'; + case 'staging-unavailable': + return 'the compatible clean staging generation was unavailable'; + case 'workspace-changed': + return 'workspace attribution changed'; + } +} diff --git a/src/code_graph/indexer_materialization.ts b/src/code_graph/indexer_materialization.ts new file mode 100644 index 00000000..aafa83d2 --- /dev/null +++ b/src/code_graph/indexer_materialization.ts @@ -0,0 +1,1537 @@ +import {Crypto, Effect, FileSystem, Option, Path} from 'effect'; +import {sha256HexSync} from '../crypto/sha256.js'; +import {SystemInfo} from '../effect/system.js'; +import {codeGraphBlobExtractionReuseClass, codeGraphBlobReuseCacheKey} from './blob_reuse.js'; +import {CODE_GRAPH_CACHE_TRANSACTION_LIMITS, codeGraphFileBlobCapacityBytes} from './cache_capacity.js'; +import { + codeGraphDiskCapacityFailure, + codeGraphPersistentCapacityDemand, + type CodeGraphDirectPersistentCapacityBoundary, +} from './disk_capacity.js'; +import { + codeGraphDiskReservationFilesystemKey, + type CodeGraphDiskReservationOptions, + withCodeGraphDiskReservation, +} from './disk_reservation.js'; +import {planCodeGraphExtractionLanes} from './extraction_lanes.js'; +import {extractRepositoryFileFacts} from './extractor.js'; +import { + budgetCachedCodeGraphFacts, + cachedCodeGraphFactBytes, + CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM, + serializeBoundedCodeGraphFact, + type BoundedCodeGraphFact, +} from './fact_budget.js'; +import {CodeGraphIndexOperationError, sameOverlayState, WorktreeChangedDuringIndex} from './indexer_shared.js'; +import type {DirectPersistentCapacityProtection} from './indexer_types.js'; +import { + worktreeBuildRequestState, + type CodeGraphContentBatchContext, + type CodeGraphInventoryOptions, +} from './inventory.js'; +import {BUILTIN_LANGUAGE_PACK_REGISTRY, type CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; +import {relocateStructuredSchemaFacts} from './languages/schemas/extractor.js'; +import {codeGraphDiskReservationLockPath, codeGraphDiskReservationRoot, type CodeGraphLayout} from './layout.js'; +import {compareCodeUnits} from './ordering.js'; +import {budgetParserWorkerFacts, type CodeGraphParserPoolShape, type CodeGraphParserResult} from './parser_worker.js'; +import { + codeGraphExtractionWorkUnits, + codeGraphSourceSizeBucket, + type CodeGraphScanningMetrics, +} from './progress_telemetry.js'; +import {repositoryIdentityMatchesExpectation, resolveRepositoryIdentity} from './repository.js'; +import { + CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION, + type CodeGraphDirectPersistentCapacityProtector, + type CodeGraphLanguagePackProvenance, + type CodeGraphStagingProgress, + type CodeGraphStoreShape, +} from './store.js'; +import {inspectCodeGraphStorage} from './storage.js'; +import {TreeSitterRuntime, type TreeSitterRuntimeShape} from './tree_sitter/runtime.js'; +import { + CODE_GRAPH_EXTRACTOR_SET_VERSION, + type CodeGraphEdge, + type CodeGraphFileFacts, + type CodeGraphInventoryFile, + type CodeGraphMaterializationRows, + type CodeGraphProgress, + type CodeGraphReference, + type CodeGraphSymbol, + type RepositoryIdentity, +} from './types.js'; + +export interface DirectPersistentCapacityContext { + readonly capacityProtection?: DirectPersistentCapacityProtection; + readonly claimMode?: CodeGraphDiskReservationOptions['claimMode']; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly layout: CodeGraphLayout; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly threadnoteHome: string; +} + +export function codeGraphDirectPersistentCapacityProtector( + input: DirectPersistentCapacityContext, +): CodeGraphDirectPersistentCapacityProtector { + return (boundary, transaction) => + input.capacityProtection + ? withCodeGraphDiskReservation( + { + boundary, + claimMode: input.claimMode, + ledgerLockPath: codeGraphDiskReservationLockPath(input.capacityProtection.path, input.threadnoteHome), + ledgerRoot: codeGraphDiskReservationRoot(input.capacityProtection.path, input.threadnoteHome), + maintenance: input.capacityProtection.maintenance + .tick({ + allowIndexPreparation: true, + anchorIdentity: input.identity, + automaticTail: false, + checkoutId: input.layout.checkoutId, + databasePath: input.layout.databasePath, + joinActive: false, + pressure: 'critical', + threadnoteHome: input.threadnoteHome, + writerLockPath: input.layout.databaseWriteLockPath, + }) + .pipe( + Effect.catch(error => (['busy', 'no-space'].includes(error.code) ? Effect.void : Effect.fail(error))), + ), + observe: observeDirectPersistentCapacity({ + boundary, + fs: input.fs, + identity: input.identity, + layout: input.layout, + protection: input.capacityProtection, + threadnoteHome: input.threadnoteHome, + }), + onDiagnostic: diagnostic => Effect.logWarning(diagnostic), + onWaiting: (input.onProgress?.({phase: 'waiting', reason: 'disk-capacity'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + }, + transaction, + ).pipe( + Effect.provideService(Crypto.Crypto, input.capacityProtection.crypto), + Effect.provideService(FileSystem.FileSystem, input.fs), + Effect.provideService(Path.Path, input.capacityProtection.path), + Effect.provideService(SystemInfo, input.capacityProtection.system), + ) + : Effect.fail( + codeGraphDiskCapacityFailure( + { + calibrationIdentity: 'direct-persistent-capacity-unavailable', + reason: 'calibration-input-unknown', + state: 'unknown', + }, + boundary.operation, + ), + ); +} + +export function promoteReadySnapshotWithCapacity( + input: DirectPersistentCapacityContext & {readonly store: CodeGraphStoreShape}, + snapshotId: string, +) { + return input.store.promote(input.layout.databasePath, input.identity, snapshotId, { + persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector(input), + }); +} + +export interface CodeGraphCacheExtractedRow { + readonly cacheFact: BoundedCodeGraphFact; + readonly cacheIdentity: string; + readonly degraded: boolean; + readonly file: CodeGraphInventoryFile; +} + +export interface CodeGraphCacheContentCoalescer { + /** @internal Accepts already-extracted rows for bounded structural/load tests. */ + readonly acceptExtracted: ( + rows: readonly CodeGraphCacheExtractedRow[], + context: CodeGraphContentBatchContext, + ) => Effect.Effect; + /** Drops references only. This is safe in failure/cancellation cleanup because it never starts a write. */ + readonly discard: Effect.Effect; + /** Flushes pending rows and is called only after inventory succeeds. */ + readonly flush: Effect.Effect; + readonly onContentBatch: NonNullable; +} + +const CODE_GRAPH_CACHE_TIMESTAMP_CAPACITY_PLACEHOLDER = '1970-01-01T00:00:00.000Z'; + +function codeGraphFileProgressDimensions( + file: CodeGraphInventoryFile, + languagePacks: CodeGraphLanguagePackRegistryShape, +) { + const matched = Option.getOrUndefined(languagePacks.match(file.path)); + return { + classifier: matched?.pack.id ?? 'unmatched', + role: matched?.role ?? 'unmatched', + sizeBucket: codeGraphSourceSizeBucket(file.size), + } as const; +} + +/** @internal Exposed for cache coalescing/cancellation contract tests. */ +export function cacheContentBatch(options: { + readonly databasePath: string; + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly onProgress?: (progress: CodeGraphProgress) => Effect.Effect; + readonly parserPool: CodeGraphParserPoolShape; + readonly persistentCapacityProtector: CodeGraphDirectPersistentCapacityProtector; + readonly store: CodeGraphStoreShape; + readonly threadnoteHome: string; + readonly treeSitter: TreeSitterRuntimeShape; +}): CodeGraphCacheContentCoalescer { + const windowSize = Math.max(1, options.parserPool.capacity * 2); + let extractionMilliseconds = 0; + let extractionFactsBytesCompleted = 0; + let extractionSourceBytesCompleted = 0; + let extractionWorkUnitsCompleted = 0; + let extractionPlan = undefined as CodeGraphContentBatchContext['extractionPlan']; + let persistenceMilliseconds = 0; + let readingMilliseconds = 0; + let pendingBytes = 0; + let pendingRows = 0; + let latestContext: CodeGraphContentBatchContext | undefined; + type PendingCacheGroup = { + readonly cacheIdentity: string; + readonly facts: BoundedCodeGraphFact[]; + readonly files: CodeGraphInventoryFile[]; + readonly paths: Set; + payloadBytes: number; + }; + const pendingGroups = new Map(); + const currentScanningMetrics = (): CodeGraphScanningMetrics | undefined => + extractionPlan === undefined + ? undefined + : { + factsBytesCompleted: extractionFactsBytesCompleted, + sourceBytesCompleted: extractionSourceBytesCompleted, + sourceBytesTotal: extractionPlan.sourceBytesTotal, + workUnitsCompleted: extractionWorkUnitsCompleted, + workUnitsTotal: extractionPlan.workUnitsTotal, + }; + const observeExtractionPlan = (plan: CodeGraphContentBatchContext['extractionPlan']) => { + if (plan === undefined) { + extractionPlan = undefined; + extractionFactsBytesCompleted = 0; + extractionSourceBytesCompleted = 0; + extractionWorkUnitsCompleted = 0; + return; + } + if ( + extractionPlan === undefined || + extractionPlan.sourceBytesTotal !== plan.sourceBytesTotal || + extractionPlan.workUnitsTotal !== plan.workUnitsTotal + ) { + extractionFactsBytesCompleted = 0; + extractionSourceBytesCompleted = 0; + extractionWorkUnitsCompleted = 0; + } + extractionPlan = plan; + }; + const completeExtractionMetrics = (file: CodeGraphInventoryFile, factsBytes: number) => { + if (extractionPlan === undefined) return undefined; + extractionFactsBytesCompleted = Math.min(Number.MAX_SAFE_INTEGER, extractionFactsBytesCompleted + factsBytes); + extractionSourceBytesCompleted = Math.min( + extractionPlan.sourceBytesTotal, + extractionSourceBytesCompleted + file.size, + ); + extractionWorkUnitsCompleted = Math.min( + extractionPlan.workUnitsTotal, + extractionWorkUnitsCompleted + + codeGraphExtractionWorkUnits(file.size, file.language, codeGraphSourceSizeBucket(file.size)), + ); + return currentScanningMetrics(); + }; + type SerializedParserResult = CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}; + const reusableExtractions = new Map(); + const reusableExtractionUses = new Map(); + const flushPendingGroup = (key: string) => + Effect.gen(function* () { + const group = pendingGroups.get(key); + if (!group || group.files.length === 0) return; + const context = latestContext; + if (!context) + return yield* Effect.fail( + new CodeGraphIndexOperationError('Code graph cache persistence context is unavailable.'), + ); + const representative = group.files[0]!; + const groupBytes = group.files.reduce((total, file) => total + file.size, 0); + const groupFactBytes = group.facts.reduce((total, fact) => total + fact.bytes, 0); + yield* emitContentProgress( + options.onProgress, + context, + { + batchCompleted: 0, + batchTotal: group.files.length, + bytes: groupBytes, + ...codeGraphFileProgressDimensions(representative, options.languagePacks), + factsBytes: groupFactBytes, + language: representative.language, + path: representative.path, + sizeBucket: codeGraphSourceSizeBucket(groupBytes), + stage: 'persisting', + }, + extractionMilliseconds, + persistenceMilliseconds, + currentScanningMetrics(), + ); + const startedAt = performance.now(); + yield* options.store.cacheFacts( + options.databasePath, + group.files, + group.facts, + group.cacheIdentity, + options.persistentCapacityProtector, + ); + const elapsed = Math.max(0, performance.now() - startedAt); + persistenceMilliseconds += elapsed; + pendingBytes -= group.payloadBytes; + pendingRows -= group.files.length; + pendingGroups.delete(key); + yield* emitContentProgress( + options.onProgress, + context, + { + batchCompleted: group.files.length, + batchTotal: group.files.length, + bytes: groupBytes, + ...codeGraphFileProgressDimensions(representative, options.languagePacks), + factsBytes: groupFactBytes, + language: representative.language, + path: representative.path, + persistMilliseconds: elapsed, + relations: group.facts.reduce((total, fact) => total + fact.facts.edges.length, 0), + sizeBucket: codeGraphSourceSizeBucket(groupBytes), + stage: 'persisting', + symbols: group.facts.reduce((total, fact) => total + fact.facts.symbols.length, 0), + }, + extractionMilliseconds, + persistenceMilliseconds, + currentScanningMetrics(), + ); + }); + const flushOldestPendingGroup = () => { + const key = pendingGroups.keys().next().value as string | undefined; + return key === undefined ? Effect.void : flushPendingGroup(key); + }; + const acceptExtracted = (rows: readonly CodeGraphCacheExtractedRow[], context: CodeGraphContentBatchContext) => + Effect.gen(function* () { + latestContext = context; + for (const {cacheFact, cacheIdentity: activeCacheIdentity, degraded, file} of rows) { + const cacheIdentity = degraded ? degradedParserCacheIdentity(activeCacheIdentity) : activeCacheIdentity; + const key = `${degraded ? 'degraded' : 'durable'}\0${cacheIdentity}`; + const reuseClass = degraded ? undefined : codeGraphBlobExtractionReuseClass(file); + const rowBytes = codeGraphFileBlobCapacityBytes({ + ...(reuseClass === undefined ? {} : {blobId: file.blobId, reuseClass}), + contentHash: file.contentHash, + createdAt: CODE_GRAPH_CACHE_TIMESTAMP_CAPACITY_PLACEHOLDER, + extractorSet: cacheIdentity, + factsJson: cacheFact.json, + path: file.path, + }); + if (rowBytes > CODE_GRAPH_CACHE_TRANSACTION_LIMITS.payloadBytes) { + return yield* Effect.fail( + new CodeGraphIndexOperationError(`Code graph cache row exceeds the persistence payload ceiling.`), + ); + } + while ( + pendingRows > 0 && + (pendingRows >= CODE_GRAPH_CACHE_TRANSACTION_LIMITS.rows || + pendingBytes > CODE_GRAPH_CACHE_TRANSACTION_LIMITS.payloadBytes - rowBytes) + ) { + yield* flushOldestPendingGroup(); + } + if (pendingGroups.get(key)?.paths.has(file.path)) { + // Committed-tree and dirty-overlay inventory phases can extract the + // same path with different content hashes. Both physical cache rows + // are reusable, so flush the older row instead of deduplicating it. + yield* flushPendingGroup(key); + } + const pending = pendingGroups.get(key) ?? { + cacheIdentity, + facts: [], + files: [], + paths: new Set(), + payloadBytes: 0, + }; + if (!pendingGroups.has(key)) pendingGroups.set(key, pending); + const {bytes: _bytes, content: _content, ...baseCacheFile} = file; + const cacheFile = degraded ? {...baseCacheFile, blobId: ''} : baseCacheFile; + pending.files.push(cacheFile); + pending.facts.push(cacheFact); + pending.paths.add(file.path); + pending.payloadBytes += rowBytes; + pendingBytes += rowBytes; + pendingRows += 1; + } + }); + const onContentBatch = ( + files: Parameters[0], + context: CodeGraphContentBatchContext, + ) => + Effect.gen(function* () { + readingMilliseconds += context.readingMilliseconds; + observeExtractionPlan(context.extractionPlan); + const cumulativeContext = {...context, readingMilliseconds}; + latestContext = cumulativeContext; + let parsedCompleted = 0; + const orderedFiles = [...files].sort((left, right) => compareCodeUnits(left.path, right.path)); + const localReuseCounts = new Map(); + for (const file of orderedFiles) { + const reuseKey = blobReuseKeyForFile(file, options.languagePacks); + if (reuseKey !== undefined) localReuseCounts.set(reuseKey, (localReuseCounts.get(reuseKey) ?? 0) + 1); + } + const expectedReuseCount = (key: string): number => + cumulativeContext.blobReuseCounts?.get(key) ?? localReuseCounts.get(key) ?? 0; + const finishReuseAttempt = (key: string | undefined) => { + if (key === undefined) return; + const uses = (reusableExtractionUses.get(key) ?? 0) + 1; + if (uses >= expectedReuseCount(key)) { + reusableExtractionUses.delete(key); + reusableExtractions.delete(key); + } else { + reusableExtractionUses.set(key, uses); + } + }; + for (const window of chunkValues(orderedFiles, windowSize)) { + let windowCompleted = 0; + const groups = extractionReuseGroups(window, options.languagePacks); + const extractGroup = (group: (typeof groups)[number]) => + Effect.forEach( + group.files, + file => + Effect.gen(function* () { + const reuseKey = group.reuseKey; + yield* emitContentProgress( + options.onProgress, + cumulativeContext, + { + batchCompleted: parsedCompleted, + batchTotal: files.length, + bytes: file.size, + ...codeGraphFileProgressDimensions(file, options.languagePacks), + language: file.language, + path: file.path, + stage: 'extracting', + }, + extractionMilliseconds, + persistenceMilliseconds, + currentScanningMetrics(), + ); + const donor = reuseKey === undefined ? undefined : reusableExtractions.get(reuseKey); + const reused = donor === undefined ? undefined : relocateSerializedParserResult(file, donor); + if (reused !== undefined) { + finishReuseAttempt(reuseKey); + windowCompleted += 1; + yield* emitContentProgress( + options.onProgress, + cumulativeContext, + { + batchCompleted: parsedCompleted + windowCompleted, + batchTotal: files.length, + bytes: file.size, + ...codeGraphFileProgressDimensions(file, options.languagePacks), + degraded: false, + factsBytes: reused.cacheFact.bytes, + language: file.language, + parseMilliseconds: 0, + path: file.path, + relations: reused.facts.edges.length, + stage: 'extracting', + symbols: reused.facts.symbols.length, + }, + extractionMilliseconds, + persistenceMilliseconds, + completeExtractionMetrics(file, reused.cacheFact.bytes), + ); + return {file, result: reused}; + } + const parsed = yield* extractParserFacts(file, options); + const cacheFact = serializeBoundedCodeGraphFact(parsed.facts); + const result = { + ...parsed, + cacheFact, + facts: cacheFact.facts, + } satisfies CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}; + if (!result.degraded && reuseKey !== undefined && expectedReuseCount(reuseKey) > 1) { + reusableExtractions.set(reuseKey, result); + } + finishReuseAttempt(reuseKey); + windowCompleted += 1; + yield* emitContentProgress( + options.onProgress, + cumulativeContext, + { + batchCompleted: parsedCompleted + windowCompleted, + batchTotal: files.length, + bytes: file.size, + ...codeGraphFileProgressDimensions(file, options.languagePacks), + degraded: result.degraded, + factsBytes: result.cacheFact.bytes, + language: file.language, + parseMilliseconds: result.parseMilliseconds, + path: file.path, + relations: result.facts.edges.length, + stage: 'extracting', + symbols: result.facts.symbols.length, + }, + extractionMilliseconds + result.parseMilliseconds, + persistenceMilliseconds, + completeExtractionMetrics(file, result.cacheFact.bytes), + ); + return {file, result}; + }), + {concurrency: 1}, + ); + const groupedResults: Array< + readonly {readonly file: CodeGraphInventoryFile; readonly result: SerializedParserResult}[] + > = []; + for (const lane of planCodeGraphExtractionLanes(groups, options.parserPool.capacity)) { + groupedResults.push(...(yield* Effect.forEach(lane.groups, extractGroup, {concurrency: lane.concurrency}))); + } + const results = groupedResults.flat(); + extractionMilliseconds += results.reduce((total, result) => total + result.result.parseMilliseconds, 0); + parsedCompleted += results.length; + const resultsByPath = new Map(results.map(result => [result.file.path, result.result])); + const extractedRows: CodeGraphCacheExtractedRow[] = []; + for (const group of groupFilesByCacheIdentity(window, options.languagePacks)) { + const durableFiles = group.files.filter(file => !resultsByPath.get(file.path)!.degraded); + const degradedFiles = group.files.filter(file => resultsByPath.get(file.path)!.degraded); + for (const [degraded, cacheFiles] of [ + [false, durableFiles], + [true, degradedFiles], + ] as const) { + for (const file of cacheFiles) { + extractedRows.push({ + cacheFact: resultsByPath.get(file.path)!.cacheFact, + cacheIdentity: group.cacheIdentity, + degraded, + file, + }); + } + } + } + yield* acceptExtracted(extractedRows, cumulativeContext); + } + }); + return { + acceptExtracted, + discard: Effect.sync(() => { + pendingGroups.clear(); + pendingBytes = 0; + pendingRows = 0; + latestContext = undefined; + reusableExtractions.clear(); + reusableExtractionUses.clear(); + }), + flush: Effect.gen(function* () { + while (pendingGroups.size > 0) yield* flushOldestPendingGroup(); + reusableExtractions.clear(); + reusableExtractionUses.clear(); + }), + onContentBatch, + }; +} + +function blobReuseKeyForFile( + file: CodeGraphInventoryFile, + languagePacks: CodeGraphLanguagePackRegistryShape, +): string | undefined { + const cacheIdentity = Option.getOrUndefined(languagePacks.cacheIdentityForPath(file.path)); + return cacheIdentity === undefined ? undefined : codeGraphBlobReuseCacheKey(file, cacheIdentity); +} + +function extractionReuseGroups( + files: readonly CodeGraphInventoryFile[], + languagePacks: CodeGraphLanguagePackRegistryShape, +): readonly {readonly files: readonly CodeGraphInventoryFile[]; readonly reuseKey?: string}[] { + const groups = new Map(); + for (const file of files) { + const reuseKey = blobReuseKeyForFile(file, languagePacks); + const key = reuseKey ?? `path\0${file.path}`; + const group = groups.get(key) ?? {files: [], ...(reuseKey === undefined ? {} : {reuseKey})}; + if (!groups.has(key)) groups.set(key, group); + group.files.push(file); + } + return [...groups.values()]; +} + +function relocateSerializedParserResult( + file: CodeGraphInventoryFile, + donor: CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}, +): (CodeGraphParserResult & {readonly cacheFact: BoundedCodeGraphFact}) | undefined { + if (donor.degraded) return undefined; + const relocated = relocateStructuredSchemaFacts(file, donor.facts); + if (relocated === undefined) return undefined; + const cacheFact = serializeBoundedCodeGraphFact(relocated); + return {cacheFact, degraded: false, facts: cacheFact.facts, parseMilliseconds: 0}; +} + +function extractParserFacts( + file: CodeGraphInventoryFile, + options: { + readonly languagePacks: CodeGraphLanguagePackRegistryShape; + readonly parserPool: CodeGraphParserPoolShape; + readonly threadnoteHome: string; + readonly treeSitter: TreeSitterRuntimeShape; + }, +): Effect.Effect { + if (file.bytes === undefined) return options.parserPool.extract(file, options.threadnoteHome); + return Effect.gen(function* () { + const startedAt = performance.now(); + const facts = yield* options.languagePacks + .extractRawFile(file) + .pipe(Effect.provideService(TreeSitterRuntime, options.treeSitter)); + const bounded = budgetParserWorkerFacts(file, facts); + return { + degraded: bounded.degraded, + facts: bounded.facts, + parseMilliseconds: Math.max(0, performance.now() - startedAt), + }; + }); +} + +function emitContentProgress( + onProgress: ((progress: CodeGraphProgress) => Effect.Effect) | undefined, + context: CodeGraphContentBatchContext, + activity: NonNullable['activity']>, + extractionMilliseconds: number, + persistenceMilliseconds: number, + metrics?: CodeGraphScanningMetrics, +) { + return ( + onProgress?.({ + ...context.progress, + activity, + ...(metrics === undefined ? {} : {metrics}), + timings: { + extractionMilliseconds, + persistenceMilliseconds, + readingMilliseconds: context.readingMilliseconds, + }, + }) ?? Effect.void + ); +} + +function chunkValues(values: readonly A[], size: number): readonly (readonly A[])[] { + const chunks: A[][] = []; + for (let index = 0; index < values.length; index += size) chunks.push(values.slice(index, index + size)); + return chunks; +} + +function degradedParserCacheIdentity(activeIdentity: string): string { + return sha256HexSync(`code-graph-parser-degraded-v1\n${activeIdentity}`); +} + +/** Cache generations that can satisfy inventory content admission for active parser packs. */ +export function codeGraphParserCacheLookupGenerations(activeIdentities: readonly string[]): readonly { + readonly activeIdentity: string; + readonly storedIdentity: string; +}[] { + return [...new Set(activeIdentities)].sort(compareCodeUnits).flatMap(activeIdentity => [ + {activeIdentity, storedIdentity: activeIdentity}, + {activeIdentity, storedIdentity: degradedParserCacheIdentity(activeIdentity)}, + ]); +} + +/** Rebind a physical cache-generation key to the active identity expected by inventory admission. */ +export function codeGraphActiveParserCacheKey(key: string, storedIdentity: string, activeIdentity: string): string { + if (storedIdentity === activeIdentity) return key; + const terminalGeneration = `\0${storedIdentity}`; + if (key.endsWith(terminalGeneration)) return `${key.slice(0, -terminalGeneration.length)}\0${activeIdentity}`; + const embeddedGeneration = `\0${storedIdentity}\0`; + return key.includes(embeddedGeneration) ? key.replace(embeddedGeneration, () => `\0${activeIdentity}\0`) : key; +} + +export const verifyIndexInput = Effect.fn('codeGraph.verifyIndexInput')(function* ( + identity: RepositoryIdentity, + verifyOverlay: boolean, + threadnoteHome: string, + requestedOverlay?: {readonly dirty: boolean; readonly fingerprint?: string}, +) { + const verifiedIdentity = yield* resolveRepositoryIdentity(identity.repoRoot); + if ( + !repositoryIdentityMatchesExpectation(verifiedIdentity, identity) || + (verifyOverlay && verifiedIdentity.headCommit !== identity.headCommit) + ) { + return yield* Effect.fail(new WorktreeChangedDuringIndex()); + } + if (!verifyOverlay) return; + if (!requestedOverlay) { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Pointer activation requires an exact worktree build request state.'), + ); + } + const verifiedOverlay = yield* worktreeBuildRequestState(verifiedIdentity, threadnoteHome); + if (!sameOverlayState(verifiedOverlay, requestedOverlay)) { + return yield* Effect.fail(new WorktreeChangedDuringIndex()); + } +}); + +export function extractorSetIdentity( + files: readonly {readonly contentHash: string; readonly path: string}[], + languagePacks: CodeGraphLanguagePackRegistryShape = BUILTIN_LANGUAGE_PACK_REGISTRY, +): string { + const paths = files.map(file => file.path); + return extractorSetIdentityFromIdentities( + languagePacks.activeCacheIdentities(paths), + languagePacks.activeDerivationIdentities(paths), + ); +} + +export function extractorSetIdentityFromPackProvenance(provenance: readonly CodeGraphLanguagePackProvenance[]): string { + return extractorSetIdentityFromIdentities( + [...new Set(provenance.map(pack => pack.cacheIdentity))], + [...new Set(provenance.map(pack => pack.derivationIdentity))], + ); +} + +function extractorSetIdentityFromIdentities( + cacheIdentities: readonly string[], + derivationIdentities: readonly string[], +): string { + const activeParsers = [...cacheIdentities].sort(compareCodeUnits).join('\n'); + const activeDerivations = [...derivationIdentities].sort(compareCodeUnits).join('\n'); + return sha256HexSync( + `${CODE_GRAPH_EXTRACTOR_SET_VERSION}\nactive-parser-packs:\n${activeParsers}\nactive-derivations:\n${activeDerivations}\nignore-policy:3\nresolution-context-policy:semantic-workspace-v1`, + ); +} + +export function parserCacheIdentity(): string { + const identity = BUILTIN_LANGUAGE_PACK_REGISTRY.cacheIdentityForPath('source.ts'); + return identity._tag === 'Some' ? identity.value : sha256HexSync(`${CODE_GRAPH_EXTRACTOR_SET_VERSION}:typescript`); +} + +export function snapshotIdentity( + identity: { + readonly headCommit: string; + readonly repositoryId: string; + readonly worktreeId: string; + }, + dirty: boolean, + extractorSet: string, + files: readonly {readonly contentHash: string; readonly path: string; readonly source: string}[], +): string { + const inventory = files + .map(file => `${file.path}\0${file.contentHash}\0${file.source}`) + .sort() + .join('\n'); + return `cgsn_${sha256HexSync( + `snapshot-v2\nlexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}\n${identity.repositoryId}\n${dirty ? identity.worktreeId : 'shared-commit'}\n${identity.headCommit}\n${dirty ? 'dirty' : 'clean'}\n${extractorSet}\n${inventory}`, + ).slice(0, 40)}`; +} + +/** + * Identifies the graph-producing inputs without coupling them to a Git commit or + * worktree. Commit observations remain snapshot rows and may safely alias this + * identity when the eligible inventory and derivation identity are unchanged. + */ +export function graphContentIdentity( + extractorSet: string, + files: readonly { + readonly contentHash: string; + readonly language?: string; + readonly mode?: string; + readonly path: string; + }[], +): string { + const inventory = files + .map(file => `${file.path}\0${file.contentHash}\0${file.language ?? ''}\0${file.mode ?? ''}`) + .sort() + .join('\n'); + return `cgc_${sha256HexSync( + `graph-content-v1\nlexical-storage:${CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION}\n${extractorSet}\n${inventory}`, + ).slice(0, 40)}`; +} + +export function directFullSnapshotIdentity(logicalSnapshotId: string): string { + if (!/^cgsn_[0-9a-f]{40}$/.test(logicalSnapshotId)) { + throw new CodeGraphIndexOperationError('Logical snapshot identity is invalid.'); + } + return `${logicalSnapshotId}-direct`; +} + +export function forcedSnapshotIdentity(logicalSnapshotId: string, forceGeneration: string | undefined): string { + return forceGeneration ? `${logicalSnapshotId}-full-${forceGeneration}` : logicalSnapshotId; +} + +export const firstReadySnapshotById = Effect.fn('codeGraph.firstReadySnapshotById')(function* ( + store: CodeGraphStoreShape, + databasePath: string, + snapshotIds: readonly string[], +) { + for (const snapshotId of snapshotIds) { + const ready = yield* store.currentLexicalReadySnapshotById(databasePath, snapshotId); + if (ready) return ready; + } + return undefined; +}); + +/** + * Decide whether a clean ready snapshot for HEAD is graph-equivalent to the + * current inventory and safe to promote without rematerializing. + * + * Requires an explicit graphContentId on the candidate so we never promote a + * same-commit row that merely shares extractor set but not inventory content. + */ +export function shouldReuseReadySnapshotForCleanCommit(input: { + readonly candidate?: { + readonly commit: string; + readonly dirty: boolean; + readonly graphContentId?: string; + readonly id: string; + }; + readonly graphContentId: string; + readonly headCommit: string; +}): boolean { + return ( + input.candidate !== undefined && + input.candidate.dirty === false && + input.candidate.commit === input.headCommit && + input.candidate.graphContentId !== undefined && + input.candidate.graphContentId === input.graphContentId + ); +} + +export const reusableReadySnapshotForCleanCommit = Effect.fn('codeGraph.reusableReadySnapshotForCleanCommit')( + function* (input: { + readonly databasePath: string; + readonly extractorSet: string; + readonly graphContentId: string; + readonly headCommit: string; + readonly repositoryId: string; + readonly store: CodeGraphStoreShape; + }) { + const candidate = yield* input.store.readySnapshotForCommit( + input.databasePath, + input.repositoryId, + input.headCommit, + input.extractorSet, + ); + return shouldReuseReadySnapshotForCleanCommit({ + candidate, + graphContentId: input.graphContentId, + headCommit: input.headCommit, + }) + ? candidate + : undefined; + }, +); + +export function embeddingSymbolSource(store: CodeGraphStoreShape, databasePath: string, snapshotId: string) { + return { + count: store.countEmbeddingSymbols(databasePath, snapshotId), + loadPage: (cursor: Parameters[2], limit: number) => + store.loadEmbeddingSymbolPage(databasePath, snapshotId, cursor, limit), + }; +} + +const observeDirectPersistentCapacity = Effect.fn('codeGraph.observeDirectPersistentCapacity')(function* (input: { + readonly boundary: CodeGraphDirectPersistentCapacityBoundary; + readonly fs: FileSystem.FileSystem; + readonly identity: RepositoryIdentity; + readonly layout: CodeGraphLayout; + readonly protection: DirectPersistentCapacityProtection; + readonly threadnoteHome: string; +}) { + const [durableFilesystem, temporaryFilesystem] = yield* Effect.all( + [ + input.fs.stat(input.layout.repositoryRoot).pipe( + Effect.map(info => info.dev), + Effect.option, + ), + input.fs.stat(input.protection.temporaryDirectory).pipe( + Effect.map(info => info.dev), + Effect.option, + ), + ] as const, + {concurrency: 2}, + ); + const filesystemsShared = + Option.isSome(durableFilesystem) && Option.isSome(temporaryFilesystem) + ? durableFilesystem.value === temporaryFilesystem.value + : undefined; + const probe = (target: string) => + input.protection.availableDiskBytes(target, input.boundary).pipe(Effect.catch(() => Effect.succeed(undefined))); + const availability = + filesystemsShared === undefined + ? Effect.succeed([undefined, undefined] as const) + : filesystemsShared + ? probe(input.layout.repositoryRoot).pipe(Effect.map(available => [available, available] as const)) + : Effect.all([probe(input.layout.repositoryRoot), probe(input.protection.temporaryDirectory)] as const, { + concurrency: 2, + }); + const [[durableAvailableBytes, temporaryAvailableBytes], storage] = yield* Effect.all( + [ + availability, + inspectCodeGraphStorage(input.threadnoteHome, input.identity.checkoutId, {openWhileLocked: true}).pipe( + Effect.option, + ), + ] as const, + {concurrency: 2}, + ); + const pageStorage = + Option.isSome(storage) && storage.value.state === 'available' && storage.value.pageStorage.state === 'available' + ? storage.value.pageStorage + : undefined; + const demand = codeGraphPersistentCapacityDemand({ + boundary: input.boundary, + lexicalFormatVersion: CODE_GRAPH_LEXICAL_COMPACT_FORMAT_VERSION, + pageSize: pageStorage?.pageSize ?? 0, + walAutoCheckpointPages: input.protection.walAutoCheckpointPages, + }); + return { + demand, + durableAvailableBytes, + durableFilesystemKey: Option.isSome(durableFilesystem) + ? (codeGraphDiskReservationFilesystemKey(input.protection.system.platform, durableFilesystem.value) ?? + 'durable-filesystem-unknown') + : 'durable-filesystem-unknown', + freelistBytes: pageStorage?.reclaimableBytes ?? 0, + temporaryAvailableBytes, + temporaryFilesystemKey: Option.isSome(temporaryFilesystem) + ? (codeGraphDiskReservationFilesystemKey(input.protection.system.platform, temporaryFilesystem.value) ?? + 'temporary-filesystem-unknown') + : 'temporary-filesystem-unknown', + }; +}); + +export function messageOf(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause); +} + +export const CODE_GRAPH_LOCK_OPTIONS = { + retryIntervalMilliseconds: 100, + staleAfterMilliseconds: 120_000, + waitTimeoutMilliseconds: Number.POSITIVE_INFINITY, +} as const; + +export const CODE_GRAPH_ACTIVATION_LEASE_MILLISECONDS = 10 * 60_000; +const FACT_MATERIALIZATION_BATCH_FILES = 128; +const FACT_MATERIALIZATION_BATCH_SOURCE_BYTES = 16 * 1_048_576; +const FACT_MATERIALIZATION_BATCH_CACHED_FACT_BYTES = CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM; +const PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES = 4; +export const PERSISTENT_MATERIALIZATION_TRANSACTION_FILES = 512; +export const PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES = 64 * 1_048_576; +export const PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES = 32 * 1_048_576; +// Conservative, warning-only planning factors informed by beta.30's observed +// production-shaped live amplification. They cover indexed TEMP rows, the durable candidate +// plus WAL, rollback/subjournals, and one concurrent worktree/repository build. +// Actual high-water telemetry remains authoritative and should recalibrate +// these factors as retained release evidence grows. +const FACT_MATERIALIZATION_TEMP_FACT_AMPLIFICATION_HEURISTIC = 5; +const FACT_MATERIALIZATION_DURABLE_FACT_AMPLIFICATION_HEURISTIC = 5; +const FACT_MATERIALIZATION_JOURNAL_FACT_AMPLIFICATION_HEURISTIC = 3; +const FACT_MATERIALIZATION_TEMP_MINIMUM_ESTIMATE_BYTES = 512 * 1_048_576; +const FACT_MATERIALIZATION_DIRECT_TEMP_ESTIMATE_BYTES = 16 * 1_048_576; +const FACT_MATERIALIZATION_DURABLE_MINIMUM_ESTIMATE_BYTES = 512 * 1_048_576; +const FACT_MATERIALIZATION_JOURNAL_MINIMUM_ESTIMATE_BYTES = 256 * 1_048_576; + +export function estimatedMaterializationStorageBytes( + factBytes: number | undefined, + sourceBytes: number, + materializationMode: 'direct-persistent' | 'temporary-staged' = 'temporary-staged', + estimateBasis: 'cached-fact-bytes' | 'final-fact-bytes' = 'cached-fact-bytes', +) { + const basisBytes = factBytes ?? sourceBytes; + const estimatedTemporaryDatabaseBytes = + materializationMode === 'direct-persistent' + ? FACT_MATERIALIZATION_DIRECT_TEMP_ESTIMATE_BYTES + : Math.max( + FACT_MATERIALIZATION_TEMP_MINIMUM_ESTIMATE_BYTES, + saturatingMultiply(basisBytes, FACT_MATERIALIZATION_TEMP_FACT_AMPLIFICATION_HEURISTIC), + ); + const estimatedDurableSnapshotBytes = Math.max( + FACT_MATERIALIZATION_DURABLE_MINIMUM_ESTIMATE_BYTES, + saturatingMultiply(basisBytes, FACT_MATERIALIZATION_DURABLE_FACT_AMPLIFICATION_HEURISTIC), + ); + const estimatedJournalBytes = Math.max( + FACT_MATERIALIZATION_JOURNAL_MINIMUM_ESTIMATE_BYTES, + saturatingMultiply(basisBytes, FACT_MATERIALIZATION_JOURNAL_FACT_AMPLIFICATION_HEURISTIC), + ); + const estimatedConcurrentBuildBytes = saturatingAdd( + estimatedTemporaryDatabaseBytes, + estimatedDurableSnapshotBytes, + estimatedJournalBytes, + ); + return { + estimateBasis: factBytes === undefined ? ('source-bytes-fallback' as const) : estimateBasis, + estimatedConcurrentBuildBytes, + estimatedDurableSnapshotBytes, + estimatedJournalBytes, + estimatedRequiredBytes: saturatingAdd(estimatedConcurrentBuildBytes, estimatedConcurrentBuildBytes), + estimatedTemporaryDatabaseBytes, + materializationMode, + }; +} + +export interface MaterializationStorageAvailability { + readonly durableAvailableBytes?: number; + readonly filesystemsShared?: boolean; + readonly temporaryAvailableBytes?: number; +} + +export type MaterializationStoragePlan = ReturnType & + MaterializationStorageAvailability & { + readonly availableBytes?: number; + readonly estimatedDurableFilesystemRequiredBytes: number; + readonly estimatedTemporaryFilesystemRequiredBytes: number; + }; + +/** + * Plans warning-only materialization headroom for SQLite's durable and TEMP + * filesystems. A second complete allowance covers one concurrent worktree or + * repository build without imposing a repository-size rejection. + */ +export function materializationStoragePlan( + estimate: ReturnType, + availability: MaterializationStorageAvailability, +): MaterializationStoragePlan { + const estimatedDurableFilesystemRequiredBytes = saturatingMultiply( + estimate.materializationMode === 'direct-persistent' + ? saturatingAdd(estimate.estimatedDurableSnapshotBytes, estimate.estimatedJournalBytes) + : estimate.estimatedDurableSnapshotBytes, + 2, + ); + const estimatedTemporaryFilesystemRequiredBytes = saturatingMultiply( + estimate.materializationMode === 'direct-persistent' + ? estimate.estimatedTemporaryDatabaseBytes + : saturatingAdd(estimate.estimatedTemporaryDatabaseBytes, estimate.estimatedJournalBytes), + 2, + ); + const sharedAvailableBytes = + availability.filesystemsShared === true + ? minimumDefined(availability.durableAvailableBytes, availability.temporaryAvailableBytes) + : undefined; + return { + ...estimate, + ...availability, + ...(sharedAvailableBytes === undefined ? {} : {availableBytes: sharedAvailableBytes}), + estimatedDurableFilesystemRequiredBytes, + estimatedTemporaryFilesystemRequiredBytes, + }; +} + +export function materializationStorageShortfalls(storage: { + readonly availableBytes?: number; + readonly durableAvailableBytes?: number; + readonly estimatedDurableFilesystemRequiredBytes?: number; + readonly estimatedRequiredBytes?: number; + readonly estimatedTemporaryFilesystemRequiredBytes?: number; + readonly filesystemsShared?: boolean; + readonly temporaryAvailableBytes?: number; +}): readonly ('durable' | 'shared' | 'temporary')[] { + if (storage.filesystemsShared === true) { + return storage.availableBytes !== undefined && + storage.estimatedRequiredBytes !== undefined && + storage.availableBytes < storage.estimatedRequiredBytes + ? ['shared'] + : []; + } + const shortfalls: ('durable' | 'temporary')[] = []; + if ( + storage.durableAvailableBytes !== undefined && + storage.estimatedDurableFilesystemRequiredBytes !== undefined && + storage.durableAvailableBytes < storage.estimatedDurableFilesystemRequiredBytes + ) { + shortfalls.push('durable'); + } + if ( + storage.temporaryAvailableBytes !== undefined && + storage.estimatedTemporaryFilesystemRequiredBytes !== undefined && + storage.temporaryAvailableBytes < storage.estimatedTemporaryFilesystemRequiredBytes + ) { + shortfalls.push('temporary'); + } + return shortfalls; +} + +function minimumDefined(left: number | undefined, right: number | undefined): number | undefined { + if (left === undefined) return right; + if (right === undefined) return left; + return Math.min(left, right); +} + +function saturatingMultiply(value: number, multiplier: number): number { + return Math.min(Number.MAX_SAFE_INTEGER, value * multiplier); +} + +function saturatingAdd(...values: readonly number[]): number { + return values.reduce((total, value) => Math.min(Number.MAX_SAFE_INTEGER, total + value), 0); +} + +export function factMaterializationBatches( + values: readonly T[], + cachedFactBytesByPath: ReadonlyMap = new Map(), +): readonly (readonly T[])[] { + const output: T[][] = []; + let batch: T[] = []; + let batchBytes = 0; + let batchFactBytes = 0; + for (const value of values) { + // Current-version cache writes and materialization reads both apply the + // same per-file compactor. Clamp defensive metadata from an unexpected + // legacy/corrupt row to that in-memory materialization ceiling, so there + // is no oversized-singleton exception in the batch planner. + const factBytes = Math.min( + CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM, + Math.max(0, cachedFactBytesByPath.get(value.path) ?? 0), + ); + if ( + batch.length > 0 && + (batch.length >= FACT_MATERIALIZATION_BATCH_FILES || + batchBytes + value.size > FACT_MATERIALIZATION_BATCH_SOURCE_BYTES || + batchFactBytes + factBytes > FACT_MATERIALIZATION_BATCH_CACHED_FACT_BYTES) + ) { + output.push(batch); + batch = []; + batchBytes = 0; + batchFactBytes = 0; + } + batch.push(value); + batchBytes += value.size; + batchFactBytes += factBytes; + } + if (batch.length > 0) output.push(batch); + return output; +} + +export interface PersistentMaterializationTransactionCandidate { + readonly factBytes: number; + readonly fileCount: number; + readonly sourceBytes: number; +} + +/** + * Coalesces contiguous, already-bounded logical receipts into larger physical + * SQLite transactions. Logical receipt identities stay unchanged so an + * interrupted build from an older release resumes without replay or graph + * drift. A candidate over a physical ceiling remains an isolated singleton. + */ +export function persistentMaterializationTransactionBatches( + values: readonly T[], + maximumBatches = PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES, +): readonly (readonly T[])[] { + const batchLimit = Math.max(1, Math.min(PERSISTENT_MATERIALIZATION_TRANSACTION_BATCHES, maximumBatches)); + const output: T[][] = []; + let batch: T[] = []; + let factBytes = 0; + let fileCount = 0; + let sourceBytes = 0; + for (const value of values) { + if ( + batch.length > 0 && + (batch.length >= batchLimit || + fileCount + value.fileCount > PERSISTENT_MATERIALIZATION_TRANSACTION_FILES || + sourceBytes + value.sourceBytes > PERSISTENT_MATERIALIZATION_TRANSACTION_SOURCE_BYTES || + factBytes + value.factBytes > PERSISTENT_MATERIALIZATION_TRANSACTION_FACT_BYTES) + ) { + output.push(batch); + batch = []; + factBytes = 0; + fileCount = 0; + sourceBytes = 0; + } + batch.push(value); + factBytes += value.factBytes; + fileCount += value.fileCount; + sourceBytes += value.sourceBytes; + } + if (batch.length > 0) output.push(batch); + return output; +} + +export function uniqueById(values: readonly T[]): readonly T[] { + const unique = new Map(); + for (const value of values) { + if (!unique.has(value.id)) unique.set(value.id, value); + } + return [...unique.values()]; +} + +/** + * Extraction may encounter the same relationship repeatedly at one call site + * or through overlapping language-pack derivations. The storage layer keeps + * strict INSERT semantics; collapse those logical duplicates deterministically + * before they reach its primary-key boundary. + */ +export function deduplicateMaterializationRelationships( + edges: readonly CodeGraphEdge[], + references: readonly CodeGraphReference[], +): { + readonly duplicateEdges: number; + readonly duplicateReferences: number; + readonly edges: readonly CodeGraphEdge[]; + readonly references: readonly CodeGraphReference[]; +} { + const edgeById = new Map(); + for (const edge of edges) { + if (!edgeById.has(edge.id)) edgeById.set(edge.id, edge); + } + const referenceByEdgeId = new Map(); + for (const reference of references) { + // Reference attribution has historically been last-wins for one logical + // edge. Preserve that contract for older, uncompacted cache rows while + // edges retain their first stable evidence occurrence. + referenceByEdgeId.set(reference.edgeId, reference); + } + return { + duplicateEdges: edges.length - edgeById.size, + duplicateReferences: references.length - referenceByEdgeId.size, + edges: [...edgeById.values()], + references: [...referenceByEdgeId.values()], + }; +} + +export function materializationRows( + symbols: readonly CodeGraphSymbol[], + edges: number, + references: readonly CodeGraphReference[], + deduplicated: {readonly edges: number; readonly references: number}, +): CodeGraphMaterializationRows { + return { + deduplicatedEdges: deduplicated.edges, + deduplicatedReferences: deduplicated.references, + edges, + lookupKeys: symbols.reduce((total, symbol) => total + (symbol.lookupKeys?.length ?? 0), 0), + referenceCandidates: references.reduce( + (total, reference) => total + reference.lookupTiers.reduce((tierTotal, tier) => tierTotal + tier.length, 0), + 0, + ), + references: references.length, + symbols: symbols.length, + }; +} + +export function addMaterializationRows( + left: CodeGraphMaterializationRows, + right: CodeGraphMaterializationRows, +): CodeGraphMaterializationRows { + return { + deduplicatedEdges: (left.deduplicatedEdges ?? 0) + (right.deduplicatedEdges ?? 0), + deduplicatedReferences: (left.deduplicatedReferences ?? 0) + (right.deduplicatedReferences ?? 0), + edges: (left.edges ?? 0) + (right.edges ?? 0), + lookupKeys: (left.lookupKeys ?? 0) + (right.lookupKeys ?? 0), + referenceCandidates: (left.referenceCandidates ?? 0) + (right.referenceCandidates ?? 0), + references: (left.references ?? 0) + (right.references ?? 0), + reexports: (left.reexports ?? 0) + (right.reexports ?? 0), + symbols: (left.symbols ?? 0) + (right.symbols ?? 0), + terms: (left.terms ?? 0) + (right.terms ?? 0), + }; +} + +export function materializationRowsWithStoreProgress( + rows: CodeGraphMaterializationRows, + progress: CodeGraphStagingProgress, +): CodeGraphMaterializationRows { + // Store observers emit a zero-row stage boundary before the first bounded + // statement. Keep the batch estimate at that boundary; replacing it with + // zero made the CLI claim that a non-empty batch contained no symbols or + // lookup keys. Positive observations monotonically replace estimates with + // the rows actually accepted by SQLite. + if (progress.rowsCompleted === 0) return rows; + switch (progress.stage) { + case 'symbols': + return {...rows, symbols: progress.rowsCompleted}; + case 'lookup-keys': + return {...rows, lookupKeys: progress.rowsCompleted}; + case 'terms': + return {...rows, terms: progress.rowsCompleted}; + case 'edges': + return {...rows, edges: progress.rowsCompleted}; + case 'references': + return {...rows, references: progress.rowsCompleted}; + case 'reference-candidates': + return {...rows, referenceCandidates: progress.rowsCompleted}; + case 'reexports': + return {...rows, reexports: progress.rowsCompleted}; + case 'analysis': + case 'receipt': + case 'validating': + case 'committing': + case 'committed': + return rows; + } +} + +interface MaterializationStorageFiles { + readonly databaseBytes: number; + readonly journalBytes: number; + readonly sharedMemoryBytes: number; + readonly totalBytes: number; + readonly walBytes: number; +} + +export function materializationStorageFiles( + fs: FileSystem.FileSystem, + databasePath: string, +): Effect.Effect { + const bytes = (file: string) => + fs.stat(file).pipe( + Effect.map(info => Math.min(Number(info.size), Number.MAX_SAFE_INTEGER)), + Effect.catch(() => Effect.succeed(0)), + ); + return Effect.all( + [bytes(databasePath), bytes(`${databasePath}-journal`), bytes(`${databasePath}-shm`), bytes(`${databasePath}-wal`)], + {concurrency: 4}, + ).pipe( + Effect.map(([databaseBytes, journalBytes, sharedMemoryBytes, walBytes]) => ({ + databaseBytes, + journalBytes, + sharedMemoryBytes, + totalBytes: databaseBytes + journalBytes + sharedMemoryBytes + walBytes, + walBytes, + })), + ); +} + +export function materializationStagingStage( + progress: CodeGraphStagingProgress, +): NonNullable['activity']>['stage'] { + switch (progress.stage) { + case 'validating': + return 'preparing-rows'; + case 'symbols': + return 'writing-symbols'; + case 'lookup-keys': + return 'writing-lookups'; + case 'terms': + return 'writing-terms'; + case 'edges': + return 'writing-edges'; + case 'reference-candidates': + return 'writing-candidates'; + case 'references': + case 'reexports': + return 'writing-references'; + case 'analysis': + return 'writing-analysis'; + case 'receipt': + return 'writing-receipt'; + case 'committing': + case 'committed': + return 'committing'; + } +} + +export function cachedFileKeys( + store: CodeGraphStoreShape, + databasePath: string, + languagePacks: CodeGraphLanguagePackRegistryShape, +): Effect.Effect, unknown> { + return Effect.forEach( + codeGraphParserCacheLookupGenerations(languagePacks.cacheIdentities), + generation => + store + .cachedCommittedFileKeys(databasePath, generation.storedIdentity) + .pipe( + Effect.map( + keys => + new Set( + [...keys].map(key => + codeGraphActiveParserCacheKey(key, generation.storedIdentity, generation.activeIdentity), + ), + ), + ), + ), + {concurrency: 1}, + ).pipe(Effect.map(sets => new Set(sets.flatMap(set => [...set])))); +} + +export function loadCachedFacts( + store: CodeGraphStoreShape, + databasePath: string, + files: readonly CodeGraphInventoryFile[], + languagePacks: CodeGraphLanguagePackRegistryShape, +): Effect.Effect< + { + readonly bytes: number; + readonly bytesByPath: ReadonlyMap; + readonly facts: ReadonlyMap; + }, + unknown +> { + return Effect.forEach( + groupFilesByCacheIdentity(files, languagePacks), + group => + Effect.gen(function* () { + const active = yield* store.loadCachedFacts(databasePath, group.files, group.cacheIdentity); + const missing = group.files.filter(file => !active.facts.has(file.path)); + if (missing.length === 0) return active; + const degraded = yield* store.loadCachedFacts( + databasePath, + missing, + degradedParserCacheIdentity(group.cacheIdentity), + ); + return { + bytes: active.bytes + degraded.bytes, + bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), + facts: new Map([...active.facts, ...degraded.facts]), + }; + }), + {concurrency: 1}, + ).pipe( + Effect.map(groups => { + const output = new Map(); + const bytesByPath = new Map(); + let bytes = 0; + for (const group of groups) { + for (const [path, facts] of group.facts) { + const persistedBytes = group.bytesByPath?.get(path); + if (persistedBytes !== undefined && persistedBytes <= CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM) { + output.set(path, facts); + bytesByPath.set(path, persistedBytes); + bytes += persistedBytes; + continue; + } + const budgeted = budgetCachedCodeGraphFacts(facts); + const budgetedBytes = cachedCodeGraphFactBytes(budgeted); + output.set(path, budgeted); + bytesByPath.set(path, budgetedBytes); + bytes += budgetedBytes; + } + } + return {bytes, bytesByPath, facts: output}; + }), + ); +} + +export function loadCachedFactsWithPackProvenance( + store: CodeGraphStoreShape, + databasePath: string, + files: readonly CodeGraphInventoryFile[], + languagePacks: CodeGraphLanguagePackRegistryShape, + provenance: readonly CodeGraphLanguagePackProvenance[], +): Effect.Effect< + { + readonly bytes: number; + readonly bytesByPath: ReadonlyMap; + readonly facts: ReadonlyMap; + }, + unknown +> { + const provenanceById = new Map(provenance.map(pack => [pack.id, pack])); + const groups = new Map(); + let unmatched = false; + for (const file of files) { + const match = Option.getOrUndefined(languagePacks.match(file.path)); + const identity = match === undefined ? undefined : provenanceById.get(match.pack.id)?.cacheIdentity; + if (identity === undefined) { + unmatched = true; + continue; + } + const group = groups.get(identity) ?? []; + group.push(file); + groups.set(identity, group); + } + if (unmatched) return Effect.succeed({bytes: 0, bytesByPath: new Map(), facts: new Map()}); + return Effect.forEach( + [...groups], + ([cacheIdentity, groupFiles]) => + Effect.gen(function* () { + const active = yield* store.loadCachedFacts(databasePath, groupFiles, cacheIdentity); + const missing = groupFiles.filter(file => !active.facts.has(file.path)); + if (missing.length === 0) return active; + const degraded = yield* store.loadCachedFacts( + databasePath, + missing, + degradedParserCacheIdentity(cacheIdentity), + ); + return { + bytes: active.bytes + degraded.bytes, + bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), + facts: new Map([...active.facts, ...degraded.facts]), + }; + }), + {concurrency: 1}, + ).pipe( + Effect.map(loaded => { + const facts = new Map(); + const bytesByPath = new Map(); + let bytes = 0; + for (const group of loaded) { + for (const [path, fact] of group.facts) { + const persistedBytes = group.bytesByPath?.get(path); + if (persistedBytes !== undefined && persistedBytes <= CODE_GRAPH_CACHED_FACT_BYTES_MAXIMUM) { + facts.set(path, fact); + bytesByPath.set(path, persistedBytes); + bytes += persistedBytes; + continue; + } + const budgeted = budgetCachedCodeGraphFacts(fact); + const budgetedBytes = cachedCodeGraphFactBytes(budgeted); + facts.set(path, budgeted); + bytesByPath.set(path, budgetedBytes); + bytes += budgetedBytes; + } + } + return {bytes, bytesByPath, facts}; + }), + ); +} + +export function cachedFactsMetadata( + store: CodeGraphStoreShape, + databasePath: string, + files: readonly CodeGraphInventoryFile[], + languagePacks: CodeGraphLanguagePackRegistryShape, +): Effect.Effect< + {readonly bytes: number; readonly bytesByPath: ReadonlyMap; readonly files: number}, + unknown +> { + return Effect.forEach( + groupFilesByCacheIdentity(files, languagePacks), + group => + Effect.gen(function* () { + const active = yield* store.loadCachedFacts(databasePath, group.files, group.cacheIdentity, {decode: false}); + const activeKeys = active.keys ?? new Set(active.facts.keys()); + const missing = group.files.filter(file => !activeKeys.has(file.path)); + if (missing.length === 0) + return {bytes: active.bytes, bytesByPath: active.bytesByPath ?? new Map(), keys: activeKeys}; + const degraded = yield* store.loadCachedFacts( + databasePath, + missing, + degradedParserCacheIdentity(group.cacheIdentity), + {decode: false}, + ); + const degradedKeys = degraded.keys ?? new Set(degraded.facts.keys()); + return { + bytes: active.bytes + degraded.bytes, + bytesByPath: new Map([...(active.bytesByPath ?? []), ...(degraded.bytesByPath ?? [])]), + keys: new Set([...activeKeys, ...degradedKeys]), + }; + }), + {concurrency: 1}, + ).pipe( + Effect.map(groups => { + const bytesByPath = new Map( + groups.flatMap(group => [...group.bytesByPath]).map(([path, bytes]) => [path, bytes] as const), + ); + return { + bytes: [...bytesByPath.values()].reduce((total, bytes) => total + bytes, 0), + bytesByPath, + files: new Set(groups.flatMap(group => [...group.keys])).size, + }; + }), + ); +} + +function groupFilesByCacheIdentity( + files: readonly T[], + languagePacks: CodeGraphLanguagePackRegistryShape, +): readonly {readonly cacheIdentity: string; readonly files: readonly T[]}[] { + const groups = new Map(); + for (const file of files) { + const matched = languagePacks.cacheIdentityForPath(file.path); + const identity = matched._tag === 'Some' ? matched.value : 'unmatched'; + const group = groups.get(identity); + if (group) group.push(file); + else groups.set(identity, [file]); + } + return [...groups] + .sort(([left], [right]) => compareCodeUnits(left, right)) + .map(([cacheIdentity, groupedFiles]) => ({cacheIdentity, files: groupedFiles})); +} diff --git a/src/code_graph/indexer_service.ts b/src/code_graph/indexer_service.ts new file mode 100644 index 00000000..6d32a357 --- /dev/null +++ b/src/code_graph/indexer_service.ts @@ -0,0 +1,923 @@ +import {Clock, Context, Crypto, Effect, FileSystem, Layer, Option, Path} from 'effect'; +import {CommandExecutor} from '../effect/command.js'; +import {SystemInfo} from '../effect/system.js'; +import {makeCodeGraphBuildReporter} from './build_status.js'; +import {isCodeGraphCapacityPause} from './disk_capacity.js'; +import {CodeGraphEmbeddingIndex} from './embedding.js'; +import { + attemptReusableDirtyBase, + buildAndActivate, + buildOwnedCleanSnapshot, + codeGraphBuildRequestKey, + completedConcurrentSnapshot, + ensureCommittedBase, + retiredSnapshotCleanupReporter, + reuseReadySnapshot, + withCodeGraphProcessLock, + withSharedCleanRequestGate, + writerSessionOptions, +} from './indexer_build.js'; +import {assessIncrementalOverlay, assessIncrementalOverlayCompatibility} from './indexer_incremental.js'; +import { + cacheContentBatch, + cachedFileKeys, + codeGraphDirectPersistentCapacityProtector, + directFullSnapshotIdentity, + extractorSetIdentity, + firstReadySnapshotById, + forcedSnapshotIdentity, + graphContentIdentity, + messageOf, + promoteReadySnapshotWithCapacity, + reusableReadySnapshotForCleanCommit, + snapshotIdentity, +} from './indexer_materialization.js'; +import { + CodeGraphIndexOperationError, + RepositoryMaintenanceInterrupted, + RepositoryRegistrationLost, + sameOverlayState, + WorktreeChangedDuringIndex, +} from './indexer_shared.js'; +import type { + CodeGraphCommitLease, + CodeGraphIndexerShape, + CodeGraphIndexOptions, + CommittedBaseResult, + DirectPersistentCapacityProtection, + IncrementalOverlayAssessment, + IncrementalOverlayPreassessment, +} from './indexer_types.js'; +import {codeGraphIndexEnsuresVectors} from './indexer_types.js'; +import {inventoryRepository, worktreeBuildRequestState} from './inventory.js'; +import {CodeGraphLanguagePackRegistry} from './languages/registry.js'; +import {codeGraphLayout} from './layout.js'; +import {runCodeGraphLifecycleOpportunity} from './lifecycle_opportunity.js'; +import {resolveAndRecordCodeGraphLocalAssociation} from './local_provenance.js'; +import {CodeGraphMaintenanceCoordinator} from './maintenance_coordinator.js'; +import {codeGraphMaintenanceIntentActive, withCodeGraphMaintenanceRegistration} from './maintenance_gate.js'; +import {CodeGraphParserPool} from './parser_worker.js'; +import {repositoryIdentityMatchesExpectation, resolveRepositoryIdentity} from './repository.js'; +import {CodeGraphStore} from './store.js'; +import {TreeSitterRuntime} from './tree_sitter/runtime.js'; +import type {CodeGraphIndexSummary, CodeGraphProgress, CodeGraphSnapshot} from './types.js'; + +export class CodeGraphIndexer extends Context.Service()( + 'threadnote/codeGraph/CodeGraphIndexer', +) { + static readonly layer = Layer.effect( + CodeGraphIndexer, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const store = yield* CodeGraphStore; + const maintenance = yield* CodeGraphMaintenanceCoordinator; + const embedding = yield* CodeGraphEmbeddingIndex; + const languagePacks = yield* CodeGraphLanguagePackRegistry; + const treeSitter = yield* TreeSitterRuntime; + const parserPool = yield* CodeGraphParserPool; + const command = yield* CommandExecutor; + const crypto = yield* Crypto.Crypto; + const system = yield* SystemInfo; + const index = (request: CodeGraphIndexOptions, attempt = 0): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const initialIdentity = yield* resolveRepositoryIdentity(request.cwd); + if ( + request.expectedIdentity && + !repositoryIdentityMatchesExpectation(initialIdentity, request.expectedIdentity) + ) { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Repository identity does not match the requested graph target.'), + ); + } + const layout = codeGraphLayout( + path, + request.threadnoteHome, + initialIdentity.checkoutId, + initialIdentity.worktreeId, + ); + const requestedOverlay = yield* worktreeBuildRequestState(initialIdentity, request.threadnoteHome); + const requestKey = request.force + ? undefined + : codeGraphBuildRequestKey(initialIdentity, requestedOverlay, languagePacks, request.incrementalOverlay); + const reporter = yield* withCodeGraphMaintenanceRegistration( + request.threadnoteHome, + Effect.gen(function* () { + if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Code graph repository root is a symbolic link.'), + ); + } + yield* fs.makeDirectory(layout.repositoryRoot, {recursive: true, mode: 0o700}); + const reporter = yield* makeCodeGraphBuildReporter( + initialIdentity, + layout, + requestKey ? {key: requestKey} : undefined, + ); + yield* request.onProgress?.({phase: 'registering'}) ?? Effect.void; + return reporter; + }), + ); + yield* Effect.forkScoped(reporter.heartbeat); + const options: CodeGraphIndexOptions = { + ...request, + onProgress: progress => + reporter.progress(progress).pipe(Effect.andThen(request.onProgress?.(progress) ?? Effect.void)), + }; + const capacityProtection: DirectPersistentCapacityProtection = { + availableDiskBytes: + options.diskCapacityAvailableBytes ?? ((target: string) => system.availableDiskBytes(target)), + crypto, + maintenance, + path, + system, + temporaryDirectory: system.tempDirectory, + walAutoCheckpointPages: options.sqliteWriterTuning?.walAutoCheckpointPages ?? 1_000, + }; + const ensureVectors = codeGraphIndexEnsuresVectors(options); + const summary = yield* withCodeGraphProcessLock( + fs, + layout.lockPath, + () => + (options.onProgress?.({phase: 'waiting', reason: 'repository-lock'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + 'index-repository', + Effect.gen(function* () { + if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Code graph repository root is a symbolic link.'), + ); + } + if (!(yield* fs.exists(layout.repositoryRoot))) { + return yield* Effect.fail(new RepositoryRegistrationLost()); + } + if (yield* codeGraphMaintenanceIntentActive(options.threadnoteHome)) { + return yield* Effect.fail(new RepositoryMaintenanceInterrupted()); + } + const build = store + .withSession( + layout.databasePath, + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + const {identity} = yield* resolveAndRecordCodeGraphLocalAssociation( + options.threadnoteHome, + options.cwd, + { + validateIdentity: identity => { + if (!repositoryIdentityMatchesExpectation(identity, initialIdentity)) { + return Effect.fail( + new CodeGraphIndexOperationError( + 'Repository identity changed while waiting for the graph lock.', + ), + ); + } + if ( + options.expectedIdentity && + !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity) + ) { + return Effect.fail( + new CodeGraphIndexOperationError( + 'Repository identity does not match the requested graph target.', + ), + ); + } + return identity.headCommit === initialIdentity.headCommit + ? Effect.void + : Effect.fail(new WorktreeChangedDuringIndex()); + }, + }, + ); + yield* store.initialize(layout.databasePath); + { + const currentOverlay = yield* worktreeBuildRequestState(identity, options.threadnoteHome); + if (!sameOverlayState(currentOverlay, requestedOverlay)) { + return yield* Effect.fail(new WorktreeChangedDuringIndex()); + } + if (requestKey) { + const completedByOwner = yield* completedConcurrentSnapshot( + store, + layout, + identity, + currentOverlay, + requestKey, + options.incrementalOverlay === false, + ); + if (completedByOwner) { + // An isolated builder exits as soon as it returns this shared result. + // Drain superseded persistent rows before that scope closes so a + // high-churn worktree cannot accumulate one full graph per request. + yield* store.retireIncompleteWorktreeSnapshots( + layout.databasePath, + identity.repositoryId, + identity.worktreeId, + new Set(), + retiredSnapshotCleanupReporter(options.onProgress), + {cleanupMode: 'required'}, + ); + yield* promoteReadySnapshotWithCapacity( + { + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + store, + threadnoteHome: options.threadnoteHome, + }, + completedByOwner.id, + ); + return yield* reuseReadySnapshot({ + embedding, + ensureVectors, + identity, + layout, + onProgress: options.onProgress, + reusedFiles: completedByOwner.fileCount, + skippedFiles: 0, + snapshot: completedByOwner, + startedAt, + store, + threadnoteHome: options.threadnoteHome, + totalFiles: completedByOwner.fileCount, + }); + } + } + } + const cachedCommittedFileKeys = options.force + ? new Set() + : yield* cachedFileKeys(store, layout.databasePath, languagePacks); + const cacheCoalescer = cacheContentBatch({ + databasePath: layout.databasePath, + languagePacks, + onProgress: options.onProgress, + parserPool, + persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + threadnoteHome: options.threadnoteHome, + }), + store, + threadnoteHome: options.threadnoteHome, + treeSitter, + }); + const inventory = yield* inventoryRepository(identity, { + ...options, + cachedCommittedFileKeys, + includeOpaqueCorpusAssets: ensureVectors, + languagePacks, + onContentBatch: cacheCoalescer.onContentBatch, + }).pipe( + Effect.tap(() => cacheCoalescer.flush), + Effect.ensuring(cacheCoalescer.discard.pipe(Effect.andThen(parserPool.trimIdle))), + ); + // Inventory and extraction build large, short-lived maps and Git payloads. Reclaim them before + // the SQLite activation phase so their heap high-water does not overlap the writer page cache. + yield* Effect.sync(() => { + Bun.gc(true); + Bun.shrink(); + }); + yield* Effect.yieldNow; + const extractorSet = extractorSetIdentity(inventory.files, languagePacks); + const graphContentId = graphContentIdentity(extractorSet, inventory.files); + const logicalSnapshotId = snapshotIdentity( + identity, + inventory.dirty, + extractorSet, + inventory.files, + ); + const forceGeneration = options.force + ? (yield* crypto.randomUUIDv4).replaceAll('-', '').slice(0, 16) + : undefined; + const forcedSnapshotId = forcedSnapshotIdentity(logicalSnapshotId, forceGeneration); + const directSnapshotId = directFullSnapshotIdentity(logicalSnapshotId); + const resumedForcedBuild = options.force + ? yield* store.resumableForcedBuild(layout.databasePath, logicalSnapshotId) + : undefined; + const readyCandidateIds = inventory.dirty + ? options.incrementalOverlay === false + ? [directSnapshotId] + : [logicalSnapshotId, directSnapshotId] + : [logicalSnapshotId]; + const existing = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); + const reusableExisting = existing + ? yield* store.currentLexicalReadySnapshotById(layout.databasePath, existing.id) + : undefined; + const reusableReadyById = !options.force + ? reusableExisting && readyCandidateIds.includes(reusableExisting.id) + ? reusableExisting + : yield* firstReadySnapshotById(store, layout.databasePath, readyCandidateIds) + : undefined; + // Exact cgsn_* can miss when inventory source/provenance differs slightly + // from the shared clean row while graph content is identical. Prefer promote + // of a HEAD-matching clean ready snapshot over rematerializing. + const reusableReady = + reusableReadyById ?? + (!options.force && !inventory.dirty + ? yield* reusableReadySnapshotForCleanCommit({ + databasePath: layout.databasePath, + extractorSet, + graphContentId, + headCommit: identity.headCommit, + repositoryId: identity.repositoryId, + store, + }) + : undefined); + // A ready candidate wins this request. Do not preserve an + // interrupted logical/direct sibling that cannot be used on + // the early-return path: a repository-sized persistent build + // would otherwise remain reachable forever unless the user + // explicitly selected that other materialization mode again. + const retainedSnapshotIds = reusableReady + ? new Set() + : options.force + ? new Set([resumedForcedBuild?.id ?? forcedSnapshotId]) + : inventory.dirty + ? new Set(readyCandidateIds) + : new Set([logicalSnapshotId]); + // Apply storage backpressure before another repository-sized materialization. + // Detached cleanup is cancelled with short-lived CLI graph builders and cannot + // keep pace with repeated WorktreeChangedDuringIndex failures. + yield* store.retireIncompleteWorktreeSnapshots( + layout.databasePath, + identity.repositoryId, + identity.worktreeId, + retainedSnapshotIds, + retiredSnapshotCleanupReporter(options.onProgress), + {cleanupMode: 'required'}, + ); + if (reusableReady) { + if (existing?.id !== reusableReady.id) { + yield* promoteReadySnapshotWithCapacity( + { + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + store, + threadnoteHome: options.threadnoteHome, + }, + reusableReady.id, + ); + } + return yield* reuseReadySnapshot({ + embedding, + ensureVectors, + identity, + layout, + onProgress: options.onProgress, + reusedFiles: inventory.files.length - inventory.parsedFiles, + skippedFiles: inventory.skipped, + snapshot: reusableReady, + startedAt, + store, + threadnoteHome: options.threadnoteHome, + totalFiles: inventory.files.length, + }); + } + if (!inventory.dirty) { + return yield* buildOwnedCleanSnapshot({ + buildOwner: reporter.ownerIdentity, + capacityProtection, + embedding, + ensureVectors, + existing, + fallbackSnapshotId: forcedSnapshotId, + force: options.force === true, + fs, + identity, + inventory, + languagePacks, + layout, + logicalSnapshotId, + onProgress: options.onProgress, + persistentMaterializationTransactionBatchLimit: + options.persistentMaterializationTransactionBatchLimit, + requestedOverlay, + startedAt, + store, + threadnoteHome: options.threadnoteHome, + }); + } + const canAttemptIncrementalOverlay = + inventory.dirty && options.incrementalOverlay !== false && options.force !== true; + const resumableDirectBuild = + inventory.dirty && !options.force + ? yield* store.resumableBuildById(layout.databasePath, directSnapshotId) + : undefined; + let workspace = inventory.workspace ?? (yield* languagePacks.discoverWorkspace(inventory.files)); + let committedBase: CommittedBaseResult | undefined; + let incrementalAssessment: IncrementalOverlayAssessment | undefined; + let incrementalPrepared = false; + let building: CodeGraphSnapshot; + let persistentOwnerToken: string | undefined; + if (resumedForcedBuild) { + building = resumedForcedBuild; + incrementalAssessment = {mode: 'fallback', reason: 'forced-full-rebuild'}; + persistentOwnerToken = yield* store.claimPersistentBuild( + layout.databasePath, + identity, + building, + {logicalSnapshotId, owner: reporter.ownerIdentity}, + ); + } else if (resumableDirectBuild) { + building = resumableDirectBuild; + incrementalAssessment = { + mode: 'fallback', + reason: options.incrementalOverlay === false ? 'disabled' : 'staging-unavailable', + }; + persistentOwnerToken = yield* store.claimPersistentBuild( + layout.databasePath, + identity, + building, + {logicalSnapshotId, owner: reporter.ownerIdentity}, + ); + } else if (!inventory.dirty && !options.force) { + building = { + commit: identity.headCommit, + dirty: false, + edgeCount: 0, + extractorSet, + fileCount: 0, + graphContentId, + id: logicalSnapshotId, + repositoryId: identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: identity.worktreeId, + }; + persistentOwnerToken = yield* store.claimPersistentBuild( + layout.databasePath, + identity, + building, + {logicalSnapshotId, owner: reporter.ownerIdentity}, + ); + } else if (!canAttemptIncrementalOverlay) { + building = { + commit: identity.headCommit, + dirty: inventory.dirty, + edgeCount: 0, + extractorSet, + fileCount: 0, + graphContentId, + id: options.force ? forcedSnapshotId : directSnapshotId, + overlayFingerprint: inventory.overlayFingerprint, + repositoryId: identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: identity.worktreeId, + }; + incrementalAssessment = { + mode: 'fallback', + reason: options.force ? 'forced-full-rebuild' : 'disabled', + }; + persistentOwnerToken = yield* store.claimPersistentBuild( + layout.databasePath, + identity, + building, + {logicalSnapshotId, owner: reporter.ownerIdentity}, + ); + } else { + const reusableDirtyBase = yield* attemptReusableDirtyBase( + { + extractorSet, + identity, + inventory, + languagePacks, + layout, + persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + threadnoteHome: options.threadnoteHome, + }), + store, + }, + workspace, + ); + let preassessment: IncrementalOverlayPreassessment; + let incrementalBuilding: CodeGraphSnapshot | undefined; + if (Option.isSome(reusableDirtyBase)) { + committedBase = reusableDirtyBase.value.committedBase; + preassessment = reusableDirtyBase.value.preassessment; + } else { + preassessment = yield* assessIncrementalOverlayCompatibility( + {extractorSet, inventory, languagePacks, layout, store}, + workspace, + ); + if (preassessment.mode === 'compatible') { + committedBase = yield* ensureCommittedBase({ + buildOwner: reporter.ownerIdentity, + capacityProtection, + embedding, + existing, + force: false, + forceGeneration, + fs, + identity, + inventory, + languagePacks, + layout, + onProgress: options.onProgress, + persistentMaterializationTransactionBatchLimit: + options.persistentMaterializationTransactionBatchLimit, + requestedOverlay, + startedAt, + store, + threadnoteHome: options.threadnoteHome, + }); + } + } + if (preassessment.mode === 'fallback') { + incrementalAssessment = preassessment; + } else { + incrementalBuilding = { + baseSnapshotId: committedBase!.snapshot.id, + commit: identity.headCommit, + dirty: inventory.dirty, + edgeCount: 0, + extractorSet, + fileCount: 0, + graphContentId, + id: logicalSnapshotId, + overlayFingerprint: inventory.overlayFingerprint, + repositoryId: identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: identity.worktreeId, + }; + incrementalAssessment = yield* assessIncrementalOverlay( + { + building: incrementalBuilding, + committedBase: committedBase!, + force: false, + incrementalOverlayEnabled: true, + inventory, + languagePacks, + layout, + store, + }, + workspace, + preassessment, + ); + } + if (incrementalAssessment.mode === 'eligible') { + if (committedBase === undefined) { + return yield* Effect.fail( + new CodeGraphIndexOperationError( + 'Incremental code graph preparation requires a committed base snapshot.', + ), + ); + } + const incrementalReusedFiles = inventory.files.length - incrementalAssessment.files.length; + const incrementalCapacityProtector = codeGraphDirectPersistentCapacityProtector({ + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + threadnoteHome: options.threadnoteHome, + }); + yield* options.onProgress?.({ + completed: 0, + phase: 'materializing', + reused: incrementalReusedFiles, + total: incrementalAssessment.files.length, + unit: 'files', + }) ?? Effect.void; + incrementalPrepared = + incrementalAssessment.reuse === 'persisted-base' + ? yield* store.preparePersistedIncrementalActivation( + layout.databasePath, + committedBase.snapshot.id, + incrementalAssessment.files, + incrementalAssessment.facts, + { + deletedPaths: incrementalAssessment.deletedPaths, + resolutionClosure: incrementalAssessment.resolutionClosure, + }, + incrementalCapacityProtector, + ) + : yield* store.replaceStagedModifiedFiles( + layout.databasePath, + committedBase.snapshot.id, + incrementalAssessment.files, + incrementalAssessment.facts, + incrementalCapacityProtector, + ); + if (!incrementalPrepared) { + incrementalAssessment = {mode: 'fallback', reason: 'staging-identity-mismatch'}; + } + } + if (incrementalPrepared && incrementalBuilding !== undefined) { + building = incrementalBuilding; + yield* store.markBuilding(layout.databasePath, identity, building); + } else { + building = { + commit: identity.headCommit, + dirty: inventory.dirty, + edgeCount: 0, + extractorSet, + fileCount: 0, + id: directSnapshotId, + overlayFingerprint: inventory.overlayFingerprint, + repositoryId: identity.repositoryId, + state: 'building', + symbolCount: 0, + worktreeId: identity.worktreeId, + }; + committedBase = undefined; + persistentOwnerToken = yield* store.claimPersistentBuild( + layout.databasePath, + identity, + building, + {logicalSnapshotId, owner: reporter.ownerIdentity}, + ); + } + } + if (incrementalPrepared) { + // The prepared delta already contains attributed facts + // and a staged workspace catalog. Retaining thousands + // of project/dependency objects through activation only + // makes one-file overlays overlap full-workspace memory + // with SQLite's effective-graph scans. + workspace = { + diagnostics: workspace.diagnostics, + fingerprint: workspace.fingerprint, + projects: [], + workspaces: [], + }; + } + return yield* buildAndActivate({ + activatePointer: true, + building, + capacityProtection, + existing, + embedding, + ensureVectors, + force: options.force === true, + fs, + identity, + inventory, + committedBase, + incrementalAssessment, + incrementalOverlayEnabled: options.incrementalOverlay !== false, + incrementalPrepared, + languagePacks, + layout, + onProgress: options.onProgress, + persistentMaterializationTransactionBatchLimit: + options.persistentMaterializationTransactionBatchLimit, + persistentOwnerToken, + requestedOverlay, + startedAt, + store, + threadnoteHome: options.threadnoteHome, + workspace, + }).pipe( + Effect.catch(cause => + persistentOwnerToken !== undefined && isCodeGraphCapacityPause(cause) + ? Effect.fail(cause) + : store + .markFailed(layout.databasePath, building.id, messageOf(cause), persistentOwnerToken) + .pipe(Effect.andThen(Effect.fail(cause))), + ), + ); + }), + writerSessionOptions(layout, options), + ) + .pipe( + Effect.tap(summary => reporter.complete(summary)), + Effect.tapError(cause => reporter.fail(cause)), + ); + return yield* withSharedCleanRequestGate({ + checkoutId: initialIdentity.checkoutId, + effect: build, + fs, + onProgress: options.onProgress, + path, + requestKey, + requestedOverlay, + threadnoteHome: options.threadnoteHome, + }); + }), + ).pipe( + Effect.ensuring( + runCodeGraphLifecycleOpportunity({ + maintenance, + opportunity: 'index-completion', + targets: [ + {anchorIdentity: initialIdentity, checkoutId: layout.checkoutId, databasePath: layout.databasePath}, + ], + threadnoteHome: request.threadnoteHome, + }).pipe(Effect.ignore), + ), + ); + return summary; + }), + ).pipe( + Effect.provideService(CommandExecutor, command), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(SystemInfo, system), + Effect.catchIf( + cause => cause instanceof WorktreeChangedDuringIndex && attempt === 0, + () => index(request, attempt + 1), + ), + ); + const ensureCommit = ( + request: Omit & {readonly commit: string}, + ) => + Effect.scoped( + Effect.gen(function* () { + const initialIdentity = yield* resolveRepositoryIdentity(request.cwd); + if ( + request.expectedIdentity && + !repositoryIdentityMatchesExpectation(initialIdentity, request.expectedIdentity) + ) { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Repository identity does not match the requested graph target.'), + ); + } + const layout = codeGraphLayout( + path, + request.threadnoteHome, + initialIdentity.checkoutId, + initialIdentity.worktreeId, + ); + const reporter = yield* withCodeGraphMaintenanceRegistration( + request.threadnoteHome, + Effect.gen(function* () { + if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Code graph repository root is a symbolic link.'), + ); + } + yield* fs.makeDirectory(layout.repositoryRoot, {recursive: true, mode: 0o700}); + return yield* makeCodeGraphBuildReporter({...initialIdentity, headCommit: request.commit}, layout); + }), + ); + yield* Effect.forkScoped(reporter.heartbeat); + const options = { + ...request, + onProgress: (progress: CodeGraphProgress) => + reporter.progress(progress).pipe(Effect.andThen(request.onProgress?.(progress) ?? Effect.void)), + }; + const capacityProtection: DirectPersistentCapacityProtection = { + availableDiskBytes: + options.diskCapacityAvailableBytes ?? ((target: string) => system.availableDiskBytes(target)), + crypto, + maintenance, + path, + system, + temporaryDirectory: system.tempDirectory, + walAutoCheckpointPages: options.sqliteWriterTuning?.walAutoCheckpointPages ?? 1_000, + }; + const lease = yield* withCodeGraphProcessLock( + fs, + layout.lockPath, + () => + (options.onProgress?.({phase: 'waiting', reason: 'repository-lock'}) ?? Effect.void).pipe( + Effect.catch(() => Effect.void), + ), + 'ensure-commit', + Effect.gen(function* () { + if ((yield* fs.readLink(layout.repositoryRoot).pipe(Effect.option))._tag === 'Some') { + return yield* Effect.fail( + new CodeGraphIndexOperationError('Code graph repository root is a symbolic link.'), + ); + } + if (!(yield* fs.exists(layout.repositoryRoot))) { + return yield* Effect.fail(new RepositoryRegistrationLost()); + } + if (yield* codeGraphMaintenanceIntentActive(options.threadnoteHome)) { + return yield* Effect.fail(new RepositoryMaintenanceInterrupted()); + } + return yield* store + .withSession( + layout.databasePath, + Effect.gen(function* () { + const {identity: currentIdentity} = yield* resolveAndRecordCodeGraphLocalAssociation( + options.threadnoteHome, + options.cwd, + { + validateIdentity: identity => { + if (!repositoryIdentityMatchesExpectation(identity, initialIdentity)) { + return Effect.fail( + new CodeGraphIndexOperationError( + 'Repository identity changed while waiting for the graph lock.', + ), + ); + } + if ( + options.expectedIdentity && + !repositoryIdentityMatchesExpectation(identity, options.expectedIdentity) + ) { + return Effect.fail( + new CodeGraphIndexOperationError( + 'Repository identity does not match the requested graph target.', + ), + ); + } + return Effect.void; + }, + }, + ); + yield* store.initialize(layout.databasePath); + const identity = {...currentIdentity, headCommit: options.commit}; + const cachedCommittedFileKeys = yield* cachedFileKeys(store, layout.databasePath, languagePacks); + const cacheCoalescer = cacheContentBatch({ + databasePath: layout.databasePath, + languagePacks, + onProgress: options.onProgress, + parserPool, + persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ + capacityProtection, + fs, + identity, + layout, + onProgress: options.onProgress, + threadnoteHome: options.threadnoteHome, + }), + store, + threadnoteHome: options.threadnoteHome, + treeSitter, + }); + const inventory = yield* inventoryRepository(identity, { + ...options, + cachedCommittedFileKeys, + includeOverlay: false, + languagePacks, + onContentBatch: cacheCoalescer.onContentBatch, + }).pipe( + Effect.tap(() => cacheCoalescer.flush), + Effect.ensuring(cacheCoalescer.discard.pipe(Effect.andThen(parserPool.trimIdle))), + ); + const committedBase = yield* ensureCommittedBase({ + buildOwner: reporter.ownerIdentity, + capacityProtection, + embedding, + force: false, + fs, + identity, + inventory, + languagePacks, + layout, + onProgress: options.onProgress, + persistentMaterializationTransactionBatchLimit: + options.persistentMaterializationTransactionBatchLimit, + startedAt: yield* Clock.currentTimeMillis, + store, + threadnoteHome: options.threadnoteHome, + }); + const snapshot = committedBase.snapshot; + const leaseToken = yield* store.acquireSnapshotLease( + layout.databasePath, + snapshot.id, + 2 * 60_000, + ); + return {leaseToken, snapshot} satisfies CodeGraphCommitLease; + }), + writerSessionOptions(layout, options), + ) + .pipe( + Effect.tap(lease => reporter.completeSnapshot(lease.snapshot)), + Effect.tapError(cause => reporter.fail(cause)), + ); + }), + ).pipe( + Effect.ensuring( + runCodeGraphLifecycleOpportunity({ + maintenance, + opportunity: 'index-completion', + targets: [ + {anchorIdentity: initialIdentity, checkoutId: layout.checkoutId, databasePath: layout.databasePath}, + ], + threadnoteHome: request.threadnoteHome, + }).pipe(Effect.ignore), + ), + ); + return lease; + }), + ).pipe( + Effect.provideService(CommandExecutor, command), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(SystemInfo, system), + ); + return CodeGraphIndexer.of({ + ensureCommit, + index: options => index(options), + }); + }), + ); +} diff --git a/src/code_graph/indexer_shared.ts b/src/code_graph/indexer_shared.ts new file mode 100644 index 00000000..5bca50f4 --- /dev/null +++ b/src/code_graph/indexer_shared.ts @@ -0,0 +1,76 @@ +import {Option} from 'effect'; +import type {CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; +import {compareCodeUnits} from './ordering.js'; +import type {CodeGraphInventoryFile} from './types.js'; + +export class CodeGraphIndexOperationError extends Error { + readonly _tag = 'CodeGraphIndexOperationError' as const; +} +export function sameOverlayState( + left: {readonly dirty: boolean; readonly fingerprint?: string}, + right: {readonly dirty: boolean; readonly fingerprint?: string}, +): boolean { + return left.dirty === right.dirty && (!left.dirty || left.fingerprint === right.fingerprint); +} + +export function sameInventoryPaths( + left: readonly CodeGraphInventoryFile[], + right: readonly CodeGraphInventoryFile[], +): boolean { + return left.length === right.length && left.every((file, index) => file.path === right[index]?.path); +} + +export function codeGraphInventoryFileChanged( + base: CodeGraphInventoryFile | undefined, + current: CodeGraphInventoryFile, + languagePacks: CodeGraphLanguagePackRegistryShape, + changedPackIds: ReadonlySet, +): boolean { + return ( + !base || + base.contentHash !== current.contentHash || + base.language !== current.language || + base.mode !== current.mode || + base.size !== current.size || + base.source !== current.source || + Option.match(languagePacks.match(current.path), { + onNone: () => false, + onSome: match => changedPackIds.has(match.pack.id), + }) + ); +} + +export function inventoryFilesForPaths( + files: readonly CodeGraphInventoryFile[], + paths: readonly string[], +): readonly CodeGraphInventoryFile[] | undefined { + const selected: CodeGraphInventoryFile[] = []; + let fileIndex = 0; + for (const path of paths) { + while (fileIndex < files.length && compareCodeUnits(files[fileIndex]!.path, path) < 0) fileIndex += 1; + const file = files[fileIndex]; + if (!file || file.path !== path) return undefined; + selected.push(file); + } + return selected; +} + +export class WorktreeChangedDuringIndex extends Error { + override readonly name = 'WorktreeChangedDuringIndex'; + + constructor() { + super('Worktree files changed during code graph indexing; retry the operation.'); + } +} + +export class RepositoryRegistrationLost extends Error { + override readonly name = 'RepositoryRegistrationLost'; +} + +export class RepositoryMaintenanceInterrupted extends Error { + override readonly name = 'RepositoryMaintenanceInterrupted'; + + constructor() { + super('Code graph indexing was superseded by repair or purge; retry the operation.'); + } +} diff --git a/src/code_graph/indexer_types.ts b/src/code_graph/indexer_types.ts new file mode 100644 index 00000000..c6913911 --- /dev/null +++ b/src/code_graph/indexer_types.ts @@ -0,0 +1,118 @@ +import {Crypto, Effect, Option, Path} from 'effect'; +import type {SystemInfoShape} from '../effect/system.js'; +import type {CodeGraphDirectPersistentCapacityBoundary} from './disk_capacity.js'; +import type {CodeGraphIncrementalWork} from './incremental_work.js'; +import type {CodeGraphInventoryOptions} from './inventory.js'; +import type {CodeGraphWorkspace} from './languages/types.js'; +import type {CodeGraphMaintenanceCoordinatorShape} from './maintenance_coordinator.js'; +import type {CodeGraphSqliteWriterSettings, CodeGraphSqliteWriterTuning} from './store.js'; +import type { + CodeGraphFileFacts, + CodeGraphIndexSummary, + CodeGraphInventoryFile, + CodeGraphOverlayFallbackReason, + CodeGraphSnapshot, + RepositoryIdentityExpectation, +} from './types.js'; + +export interface CodeGraphIndexOptions extends CodeGraphInventoryOptions { + readonly cwd: string; + /** When false, skip blocking vector materialization after a ready structural snapshot. */ + readonly ensureVectors?: boolean; + /** Exact graph target supplied by a trusted local administration surface. */ + readonly expectedIdentity?: RepositoryIdentityExpectation; + readonly force?: boolean; + /** Internal benchmark/correctness escape hatch; normal indexing keeps this enabled. */ + readonly incrementalOverlay?: boolean; + /** @internal Records read-back PRAGMA values for controlled benchmark evidence. */ + readonly onSqliteWriterConfigured?: (settings: CodeGraphSqliteWriterSettings) => Effect.Effect; + /** @internal Benchmark-only physical transaction grouping; normal indexing uses four logical receipts. */ + readonly persistentMaterializationTransactionBatchLimit?: 1 | 4; + /** @internal Benchmark-only SQLite writer candidate; normal indexing leaves this unset. */ + readonly sqliteWriterTuning?: CodeGraphSqliteWriterTuning; + /** @internal Deterministic fresh-capacity probe used by lifecycle fault tests. */ + readonly diskCapacityAvailableBytes?: ( + path: string, + boundary: CodeGraphDirectPersistentCapacityBoundary, + ) => Effect.Effect; + readonly threadnoteHome: string; +} + +export interface DirectPersistentCapacityProtection { + readonly availableDiskBytes: ( + path: string, + boundary: CodeGraphDirectPersistentCapacityBoundary, + ) => Effect.Effect; + readonly crypto: Crypto.Crypto; + readonly maintenance: CodeGraphMaintenanceCoordinatorShape; + readonly path: Path.Path; + readonly system: SystemInfoShape; + readonly temporaryDirectory: string; + readonly walAutoCheckpointPages: number; +} + +export function codeGraphIndexEnsuresVectors(options: {readonly ensureVectors?: boolean}): boolean { + return options.ensureVectors !== false; +} + +export interface CommittedBaseResult { + readonly diagnostics: readonly string[]; + readonly leaseToken: Option.Option; + readonly snapshot: CodeGraphSnapshot; + readonly stagingReusable: boolean; +} + +export type IncrementalOverlayAssessment = + | { + readonly facts: readonly CodeGraphFileFacts[]; + readonly files: readonly CodeGraphInventoryFile[]; + readonly closureProjects?: number; + readonly mode: 'eligible'; + readonly deletedPaths?: readonly string[]; + readonly resolutionClosure?: 'changed' | 'full' | 'project'; + readonly extractorTransition?: true; + readonly reuse: 'persisted-base' | 'staged-base'; + readonly work: CodeGraphIncrementalWork; + } + | { + readonly mode: 'fallback'; + readonly reason: CodeGraphOverlayFallbackReason; + }; + +export type IncrementalOverlayPreassessment = + | { + readonly committedWorkspace: CodeGraphWorkspace; + readonly facts: readonly CodeGraphFileFacts[]; + readonly files: readonly CodeGraphInventoryFile[]; + readonly closureProjects?: number; + readonly mode: 'compatible'; + readonly deletedPaths?: readonly string[]; + readonly resolutionClosure?: 'changed' | 'full' | 'project'; + readonly extractorTransition?: true; + } + | { + readonly mode: 'fallback'; + readonly reason: CodeGraphOverlayFallbackReason; + }; + +export type ReusableCleanSnapshotAttempt = + | { + readonly mode: 'complete'; + readonly summary: CodeGraphIndexSummary; + } + | { + readonly mode: 'fallback'; + readonly reason: CodeGraphOverlayFallbackReason; + }; + +export interface CodeGraphCommitLease { + readonly leaseToken: string; + readonly snapshot: CodeGraphSnapshot; +} + +export interface CodeGraphIndexerShape { + readonly ensureCommit: ( + options: Omit & {readonly commit: string}, + ) => Effect.Effect; + readonly index: (options: CodeGraphIndexOptions) => Effect.Effect; +} diff --git a/src/code_graph/inventory.ts b/src/code_graph/inventory.ts index 78f20ea0..22c626b3 100644 --- a/src/code_graph/inventory.ts +++ b/src/code_graph/inventory.ts @@ -1,11 +1,25 @@ import {Effect, FileSystem, Option, Path} from 'effect'; import {sha256HexSync} from '../crypto/sha256.js'; import {runBinaryCommandEffect, runCommandEffect} from '../effect/command.js'; -import {SystemInfo} from '../effect/system.js'; import {codeGraphBlobReuseCacheKey} from './blob_reuse.js'; +import { + inspectContainedStableRegularFile, + materializeContainedStableRegularFile, + readOptionalText, + type StableContainedRegularFileMetadata, +} from './inventory_contained_file.js'; +import { + acceptsBinaryContent, + appearsBinary, + appearsGitLfsPointer, + decodeUtf8, + repositoryContentOmissionReason, + retainResolutionContext, + shouldOmitRepositoryContent, +} from './inventory_content.js'; +import {CodeGraphInventoryError} from './inventory_error.js'; import {BUILTIN_LANGUAGE_PACK_REGISTRY, type CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; import {CORPUS_EXTRACTION_SOURCE_BYTES_LIMIT, isOpaqueCorpusMediaPath} from './languages/corpus/policy.js'; -import {isLowSignalStructuredPath} from './languages/schemas/policy.js'; import type {CodeGraphFileRole, CodeGraphWorkspace} from './languages/types.js'; import { codeGraphInventoryExclusionReason, @@ -22,6 +36,8 @@ import { import {type CodeGraphInventoryFile, type CodeGraphProgress, type RepositoryIdentity} from './types.js'; export {codeGraphInventoryExclusionReason} from './inventory_policy.js'; +export {readContainedStableRegularFile, type ContainedReadInterlock} from './inventory_contained_file.js'; +export {shouldOmitRepositoryContent} from './inventory_content.js'; interface GitTreeEntry { readonly blobId: string; @@ -190,20 +206,6 @@ const GENERATED_DIRECTORIES = new Set([ const AUTHORED_DOT_DIRECTORIES = new Set(['.aspect']); const CAT_FILE_BATCH_ENTRIES = 128; const CAT_FILE_BATCH_BYTES = 16 * 1_048_576; -const COMPACT_RESOLUTION_CONTEXT_NAMES = new Set([ - 'build.gradle', - 'build.gradle.kts', - 'go.mod', - 'gradle.properties', - 'package.json', - 'package.swift', - 'pom.xml', - 'project.pbxproj', - 'settings.gradle', - 'settings.gradle.kts', - 'tsconfig.json', -]); - /** * Aggregate path/size metadata through the same admission rules used by the * inventory reader. The result is deliberately path-free and content-free. @@ -1322,22 +1324,22 @@ export function parseGitCatFileBatch( let offset = 0; for (const expected of entries) { const newline = bytes.indexOf(10, offset); - if (newline < 0) throw new Error('Git cat-file batch ended before its header.'); + if (newline < 0) throw new CodeGraphInventoryError('Git cat-file batch ended before its header.'); const header = new TextDecoder().decode(bytes.subarray(offset, newline)); const match = /^([0-9a-f]+) blob (\d+)$/.exec(header); if (!match || match[1] !== expected.blobId) { - throw new Error(`Git cat-file returned an unexpected object for ${expected.blobId}.`); + throw new CodeGraphInventoryError(`Git cat-file returned an unexpected object for ${expected.blobId}.`); } const size = Number(match[2]); const start = newline + 1; const end = start + size; if (!Number.isSafeInteger(size) || size < 0 || end >= bytes.byteLength || bytes[end] !== 10) { - throw new Error(`Git cat-file returned a truncated object for ${expected.blobId}.`); + throw new CodeGraphInventoryError(`Git cat-file returned a truncated object for ${expected.blobId}.`); } output.push(bytes.slice(start, end)); offset = end + 1; } - if (offset !== bytes.byteLength) throw new Error('Git cat-file batch returned trailing bytes.'); + if (offset !== bytes.byteLength) throw new CodeGraphInventoryError('Git cat-file batch returned trailing bytes.'); return output; } @@ -1718,293 +1720,6 @@ function chunkTreeEntries(entries: readonly T[]): readon return batches; } -function retainResolutionContext( - file: CodeGraphInventoryFile, - languagePacks: CodeGraphLanguagePackRegistryShape, -): CodeGraphInventoryFile { - const name = file.path.split('/').at(-1)?.toLowerCase() ?? ''; - const content = - file.content === undefined - ? undefined - : (compactResolutionContext(name, file.content) ?? - (languagePacks.isResolutionContext(file.path) && !COMPACT_RESOLUTION_CONTEXT_NAMES.has(name) - ? file.content - : undefined)); - if (content !== undefined) { - return {...file, content}; - } - const {bytes: _bytes, content: _content, contentOmittedReason: _contentOmittedReason, ...metadata} = file; - return metadata; -} - -function isCorpusContent(path: string, languagePacks: CodeGraphLanguagePackRegistryShape): boolean { - return Option.match(languagePacks.match(path), { - onNone: () => false, - onSome: value => value.role === 'corpus', - }); -} - -export function shouldOmitRepositoryContent( - path: string, - size: number, - languagePacks: CodeGraphLanguagePackRegistryShape = BUILTIN_LANGUAGE_PACK_REGISTRY, -): boolean { - return repositoryContentOmissionReason(path, size, languagePacks) !== undefined; -} - -function repositoryContentOmissionReason( - path: string, - size: number, - languagePacks: CodeGraphLanguagePackRegistryShape, -): CodeGraphInventoryFile['contentOmittedReason'] { - const match = languagePacks.match(path); - if (Option.isNone(match)) return undefined; - if ( - (match.value.language === 'json' || match.value.language === 'jsonc' || match.value.language === 'yaml') && - isLowSignalStructuredPath(path) - ) { - return 'metadata-only'; - } - if (match.value.role !== 'corpus') return undefined; - if (size > CORPUS_EXTRACTION_SOURCE_BYTES_LIMIT) return 'size-budget'; - return match.value.language === 'image' || match.value.language === 'audio' || match.value.language === 'video' - ? 'metadata-only' - : undefined; -} - -function acceptsBinaryContent(path: string, languagePacks: CodeGraphLanguagePackRegistryShape): boolean { - return isCorpusContent(path, languagePacks); -} - -function compactResolutionContext(name: string, content: string): string | undefined { - if (name === 'go.mod') return compactGoModule(content); - if (name === 'pom.xml') return compactMavenManifest(content); - if (name === 'settings.gradle' || name === 'settings.gradle.kts') return compactGradleSettings(content); - if (name === 'build.gradle' || name === 'build.gradle.kts') return compactGradleBuild(content); - if (name === 'gradle.properties') return ''; - if (name === 'package.swift') return compactSwiftPackage(content); - if (name === 'project.pbxproj') return compactXcodeProject(content); - if (name !== 'package.json' && name !== 'tsconfig.json') return undefined; - try { - const parsed = JSON.parse(content) as Record; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined; - if (name === 'package.json') { - const entry = packageEntryForResolution(parsed.exports, parsed.main); - const dependencySections = Object.fromEntries( - ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies'].flatMap(section => { - const value = parsed[section]; - if (!value || typeof value !== 'object' || Array.isArray(value)) return []; - return [ - [ - section, - Object.fromEntries( - Object.entries(value as Record).flatMap(([dependency, version]) => - typeof version === 'string' ? [[dependency, version]] : [], - ), - ), - ], - ]; - }), - ); - return JSON.stringify({ - ...dependencySections, - ...(entry === undefined ? {} : {main: entry}), - ...(typeof parsed.name === 'string' ? {name: parsed.name} : {}), - ...(typeof parsed.packageManager === 'string' ? {packageManager: parsed.packageManager} : {}), - ...(Array.isArray(parsed.workspaces) - ? {workspaces: parsed.workspaces.filter((value): value is string => typeof value === 'string')} - : parsed.workspaces && typeof parsed.workspaces === 'object' - ? { - workspaces: { - packages: Array.isArray((parsed.workspaces as Record).packages) - ? ((parsed.workspaces as Record).packages as unknown[]).filter( - (value): value is string => typeof value === 'string', - ) - : [], - }, - } - : {}), - }); - } - const compilerOptions = - parsed.compilerOptions && typeof parsed.compilerOptions === 'object' && !Array.isArray(parsed.compilerOptions) - ? (parsed.compilerOptions as Record) - : {}; - const paths = - compilerOptions.paths && typeof compilerOptions.paths === 'object' && !Array.isArray(compilerOptions.paths) - ? Object.fromEntries( - Object.entries(compilerOptions.paths as Record).flatMap(([alias, targets]) => - Array.isArray(targets) - ? [[alias, targets.filter((target): target is string => typeof target === 'string')]] - : [], - ), - ) - : undefined; - const compact: Record = { - compilerOptions: { - ...(typeof compilerOptions.baseUrl === 'string' ? {baseUrl: compilerOptions.baseUrl} : {}), - ...(typeof compilerOptions.outDir === 'string' ? {outDir: compilerOptions.outDir} : {}), - ...(paths === undefined ? {} : {paths}), - }, - }; - for (const field of ['exclude', 'files', 'include'] as const) { - if (Object.prototype.hasOwnProperty.call(parsed, field)) { - compact[field] = Array.isArray(parsed[field]) - ? parsed[field].filter((value): value is string => typeof value === 'string') - : parsed[field]; - } - } - if (Array.isArray(parsed.references)) { - compact.references = parsed.references.flatMap(reference => - reference && - typeof reference === 'object' && - !Array.isArray(reference) && - typeof (reference as Record).path === 'string' - ? [{path: (reference as Record).path}] - : [], - ); - } - return JSON.stringify(compact); - } catch { - return undefined; - } -} - -function compactGoModule(content: string): string { - const output: string[] = []; - let inRequireBlock = false; - for (const rawLine of content.split(/\r?\n/)) { - const line = rawLine.replace(/\/\/.*$/, '').trim(); - if (!line) continue; - if (/^module\s+\S+/.test(line)) { - output.push(line); - continue; - } - if (/^require\s*\($/.test(line)) { - inRequireBlock = true; - continue; - } - if (inRequireBlock && line === ')') { - inRequireBlock = false; - continue; - } - if (inRequireBlock && /^\S+\s+v\S+/.test(line)) { - output.push(line); - continue; - } - if (/^require\s+\S+\s+v\S+/.test(line)) output.push(line); - } - return `${output.join('\n')}\n`; -} - -function compactMavenManifest(content: string): string | undefined { - const project = content.replace(//i, ''); - const group = compactXmlTag(project, 'groupId'); - const artifact = compactXmlTag(project, 'artifactId'); - if (!artifact) return undefined; - const modules = compactXmlTags(content, 'module'); - const dependencies = [...content.matchAll(//gi)].flatMap(match => { - const dependencyArtifact = compactXmlTag(match[0], 'artifactId'); - if (!dependencyArtifact) return []; - const dependencyGroup = compactXmlTag(match[0], 'groupId'); - return [ - `${dependencyGroup ? `${dependencyGroup}` : ''}${dependencyArtifact}`, - ]; - }); - return [ - '', - group ? `${group}` : '', - `${artifact}`, - modules.length > 0 ? `${modules.map(module => `${module}`).join('')}` : '', - dependencies.length > 0 ? `${dependencies.join('')}` : '', - '', - ].join(''); -} - -function compactGradleSettings(content: string): string { - return `${content - .split(/\r?\n/) - .filter(line => /\brootProject\.name\b|^\s*include\b|\.projectDir\s*=/.test(line)) - .join('\n')}\n`; -} - -function compactGradleBuild(content: string): string { - return `${content - .split(/\r?\n/) - .filter(line => /\bproject\s*\(/.test(line)) - .join('\n')}\n`; -} - -function compactSwiftPackage(content: string): string | undefined { - const packageName = /\bPackage\s*\(\s*name\s*:\s*"([^"]+)"/m.exec(content)?.[1]; - const starts = [...content.matchAll(/\.(target|executableTarget|testTarget)\s*\(\s*name\s*:\s*"([^"]+)"/g)]; - if (!packageName && starts.length === 0) return undefined; - const targets = starts.map((match, index) => { - const body = content.slice(match.index, starts[index + 1]?.index ?? content.length); - const path = /\bpath\s*:\s*"([^"]+)"/.exec(body)?.[1]; - const dependencies = /\bdependencies\s*:\s*\[([\s\S]*?)\]/.exec(body)?.[1] ?? ''; - const names = [...dependencies.matchAll(/"([^"]+)"/g)].map(value => value[1]!); - return `.${match[1]}(name: ${JSON.stringify(match[2])}, dependencies: [${names - .map(name => JSON.stringify(name)) - .join(', ')}]${path ? `, path: ${JSON.stringify(path)}` : ''})`; - }); - return `let package = Package(name: ${JSON.stringify(packageName ?? 'Package')}, targets: [${targets.join(', ')}])\n`; -} - -function compactXcodeProject(content: string): string { - const targets = [...content.matchAll(/isa\s*=\s*PBXNativeTarget;[\s\S]*?\bname\s*=\s*"?([^";\n]+)"?;/g)].map(match => - match[1]!.trim(), - ); - return `${targets.map(target => `isa = PBXNativeTarget; name = ${JSON.stringify(target)};`).join('\n')}\n`; -} - -function compactXmlTag(content: string, tag: string): string | undefined { - return new RegExp(`<${tag}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*`, 'i').exec(content)?.[1]?.trim(); -} - -function compactXmlTags(content: string, tag: string): readonly string[] { - return [...content.matchAll(new RegExp(`<${tag}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*`, 'gi'))].map(match => - match[1]!.trim(), - ); -} - -function packageEntryForResolution(exportsValue: unknown, mainValue: unknown): string | undefined { - if (exportsValue === undefined) return typeof mainValue === 'string' ? mainValue : undefined; - const root = - typeof exportsValue === 'object' && - exportsValue !== null && - !Array.isArray(exportsValue) && - Object.keys(exportsValue).some(key => key.startsWith('.')) - ? (exportsValue as Record)['.'] - : exportsValue; - const targets = new Set(collectResolutionExportTargets(root)); - return targets.size === 1 ? [...targets][0] : undefined; -} - -function collectResolutionExportTargets(value: unknown): readonly string[] { - if (typeof value === 'string') return [value]; - if (Array.isArray(value)) return value.flatMap(collectResolutionExportTargets); - if (typeof value !== 'object' || value === null) return []; - return Object.values(value as Record).flatMap(collectResolutionExportTargets); -} - -function appearsBinary(bytes: Uint8Array): boolean { - return bytes.subarray(0, Math.min(bytes.byteLength, 8192)).includes(0); -} - -function appearsGitLfsPointer(bytes: Uint8Array): boolean { - if (bytes.byteLength > 1024) return false; - return new TextDecoder().decode(bytes).startsWith('version https://git-lfs.github.com/spec/v1\n'); -} - -function decodeUtf8(bytes: Uint8Array): string | undefined { - try { - return new TextDecoder('utf-8', {fatal: true}).decode(bytes); - } catch { - return undefined; - } -} - function normalizeRepositoryPath(value: string): string { return value.replace(/^\.\/+/, ''); } @@ -2033,334 +1748,3 @@ function cacheKey(path: string, contentHash: string, languagePacks: CodeGraphLan function isZeroObjectId(value: string): boolean { return /^0{40}(?:0{24})?$/.test(value); } - -const readOptionalText = Effect.fn('codeGraph.readOptionalText')(function* (fs: FileSystem.FileSystem, target: string) { - const opened = yield* readStableRegularFile(fs, target).pipe(Effect.option); - return opened._tag === 'Some' ? (decodeUtf8(opened.value.bytes) ?? '') : ''; -}); - -interface StableRegularFile { - readonly bytes: Uint8Array; - readonly identity: FileSystem.File.Info; - readonly openedPath: Option.Option; -} - -export interface ContainedReadInterlock { - readonly afterOpen?: Effect.Effect; - readonly beforeOpen?: Effect.Effect; -} - -function readStableRegularFile( - fs: FileSystem.FileSystem, - target: string, - interlock?: ContainedReadInterlock, -): Effect.Effect { - return Effect.gen(function* () { - const linkTarget = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - if (Option.isSome(linkTarget)) { - return yield* Effect.fail(new Error(`Refusing to read a symbolic repository file: ${target}`)); - } - const pathInfoBefore = yield* fs.stat(target); - if (pathInfoBefore.type !== 'File') { - return yield* Effect.fail(new Error(`Refusing to read a non-regular repository file: ${target}`)); - } - yield* interlock?.beforeOpen ?? Effect.void; - return yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(target, {flag: 'r'}); - yield* interlock?.afterOpen ?? Effect.void; - const openedInfoBefore = yield* file.stat; - const openedPath = yield* openedFilePath(fs, file); - const pathInfoOpened = yield* fs.stat(target); - if (!sameRegularFile(pathInfoBefore, pathInfoOpened, openedInfoBefore)) { - return yield* Effect.fail(new Error(`Repository file changed while it was opened: ${target}`)); - } - const byteLength = Number(openedInfoBefore.size); - if (!Number.isSafeInteger(byteLength) || byteLength < 0) { - return yield* Effect.fail(new Error(`Repository file size cannot be represented safely: ${target}`)); - } - const bytes = new Uint8Array(byteLength); - let offset = 0; - while (offset < bytes.byteLength) { - const read = Number(yield* file.read(bytes.subarray(offset))); - if (read <= 0) { - return yield* Effect.fail(new Error(`Repository file ended while it was being read: ${target}`)); - } - offset += read; - } - const openedInfoAfter = yield* file.stat; - const linkTargetAfter = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - if (Option.isSome(linkTargetAfter)) { - return yield* Effect.fail(new Error(`Repository file became a symbolic link while reading: ${target}`)); - } - const pathInfoAfter = yield* fs.stat(target); - if ( - !sameRegularFile(pathInfoBefore, pathInfoAfter, openedInfoAfter) || - openedInfoBefore.size !== openedInfoAfter.size - ) { - return yield* Effect.fail(new Error(`Repository file changed while it was being read: ${target}`)); - } - return {bytes, identity: openedInfoAfter, openedPath}; - }), - ); - }).pipe(Effect.mapError(cause => new Error(`Could not safely read repository file ${target}.`, {cause}))); -} - -export function readContainedStableRegularFile( - fs: FileSystem.FileSystem, - path: Path.Path, - repositoryRoot: string, - relative: string, - interlock?: ContainedReadInterlock, -): Effect.Effect { - const target = path.join(repositoryRoot, ...relative.split('/')); - return Effect.gen(function* () { - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalBefore = yield* fs.realPath(target); - if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { - return yield* Effect.fail(new Error(`Repository file resolves outside its root: ${relative}`)); - } - const opened = yield* readStableRegularFile(fs, target, interlock); - if (Option.isSome(opened.openedPath) && !isContainedPath(path, repositoryRoot, opened.openedPath.value)) { - return yield* Effect.fail(new Error(`Opened repository file is outside its root: ${relative}`)); - } - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalAfter = yield* fs.realPath(target); - const finalInfo = yield* fs.stat(target); - if (!isContainedPath(path, repositoryRoot, canonicalAfter)) { - return yield* Effect.fail(new Error(`Repository file escaped its root while reading: ${relative}`)); - } - if (!sameRegularFile(opened.identity, finalInfo, opened.identity)) { - return yield* Effect.fail(new Error(`Repository path no longer identifies the opened file: ${relative}`)); - } - return opened.bytes; - }).pipe(Effect.mapError(cause => new Error(`Could not safely read repository path ${relative}.`, {cause}))); -} - -interface StableContainedMaterialization { - readonly bytes?: Uint8Array; - readonly contentHash: string; - readonly size: number; -} - -interface StableContainedRegularFileMetadata { - readonly size: number; -} - -/** - * Inspect only stable, contained path metadata. Admission depends on path and - * size, so excluded dirty files never need to be opened, read, or hashed. - */ -function inspectContainedStableRegularFile( - fs: FileSystem.FileSystem, - path: Path.Path, - repositoryRoot: string, - relative: string, -): Effect.Effect { - const target = path.join(repositoryRoot, ...relative.split('/')); - return Effect.gen(function* () { - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalBefore = yield* fs.realPath(target); - if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { - return yield* Effect.fail(new Error(`Repository file resolves outside its root: ${relative}`)); - } - const linkBefore = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - if (Option.isSome(linkBefore)) { - return yield* Effect.fail(new Error(`Refusing to inspect a symbolic repository file: ${relative}`)); - } - const infoBefore = yield* fs.stat(target); - const size = Number(infoBefore.size); - if (infoBefore.type !== 'File' || !Number.isSafeInteger(size) || size < 0) { - return yield* Effect.fail(new Error(`Repository file metadata is not safely representable: ${relative}`)); - } - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalAfter = yield* fs.realPath(target); - const linkAfter = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - const infoAfter = yield* fs.stat(target); - if ( - Option.isSome(linkAfter) || - !isContainedPath(path, repositoryRoot, canonicalAfter) || - !sameRegularFile(infoBefore, infoAfter, infoBefore) || - infoBefore.size !== infoAfter.size - ) { - return yield* Effect.fail(new Error(`Repository file changed while its metadata was inspected: ${relative}`)); - } - return {size}; - }).pipe(Effect.mapError(cause => new Error(`Could not safely inspect repository path ${relative}.`, {cause}))); -} - -/** - * Safely materialize a worktree file, or hash it through a fixed-size buffer when - * policy says its content should remain metadata-only. This keeps dirty large - * corpus artifacts from allocating their full size while preserving an exact - * content fingerprint and the same symlink/race interlocks as ordinary reads. - */ -function materializeContainedStableRegularFile( - fs: FileSystem.FileSystem, - path: Path.Path, - repositoryRoot: string, - relative: string, - omitContent: (size: number) => boolean, - expectedSize?: number, -): Effect.Effect { - const target = path.join(repositoryRoot, ...relative.split('/')); - return Effect.gen(function* () { - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalBefore = yield* fs.realPath(target); - if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { - return yield* Effect.fail(new Error(`Repository file resolves outside its root: ${relative}`)); - } - const linkTarget = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - if (Option.isSome(linkTarget)) { - return yield* Effect.fail(new Error(`Refusing to read a symbolic repository file: ${relative}`)); - } - const pathInfoBefore = yield* fs.stat(target); - if (pathInfoBefore.type !== 'File') { - return yield* Effect.fail(new Error(`Refusing to read a non-regular repository file: ${relative}`)); - } - const materialized = yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(target, {flag: 'r'}); - const openedInfoBefore = yield* file.stat; - const openedPath = yield* openedFilePath(fs, file); - const pathInfoOpened = yield* fs.stat(target); - if (!sameRegularFile(pathInfoBefore, pathInfoOpened, openedInfoBefore)) { - return yield* Effect.fail(new Error(`Repository file changed while it was opened: ${relative}`)); - } - if (Option.isSome(openedPath) && !isContainedPath(path, repositoryRoot, openedPath.value)) { - return yield* Effect.fail(new Error(`Opened repository file is outside its root: ${relative}`)); - } - const size = Number(openedInfoBefore.size); - if (!Number.isSafeInteger(size) || size < 0) { - return yield* Effect.fail(new Error(`Repository file size cannot be represented safely: ${relative}`)); - } - if (expectedSize !== undefined && size !== expectedSize) { - return yield* Effect.fail(new Error(`Repository file size changed before it was read: ${relative}`)); - } - const hasher = new Bun.CryptoHasher('sha256'); - const bytes = omitContent(size) ? undefined : new Uint8Array(size); - const buffer = bytes ?? new Uint8Array(Math.min(1_048_576, Math.max(1, size))); - let offset = 0; - while (offset < size) { - const view = bytes ? bytes.subarray(offset) : buffer.subarray(0, Math.min(buffer.byteLength, size - offset)); - const read = Number(yield* file.read(view)); - if (read <= 0) { - return yield* Effect.fail(new Error(`Repository file ended while it was being read: ${relative}`)); - } - hasher.update(view.subarray(0, read)); - offset += read; - } - const openedInfoAfter = yield* file.stat; - const linkTargetAfter = yield* fs.readLink(target).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - const pathInfoAfter = yield* fs.stat(target); - if ( - Option.isSome(linkTargetAfter) || - !sameRegularFile(pathInfoBefore, pathInfoAfter, openedInfoAfter) || - openedInfoBefore.size !== openedInfoAfter.size - ) { - return yield* Effect.fail(new Error(`Repository file changed while it was being read: ${relative}`)); - } - return {bytes, contentHash: hasher.digest('hex'), size} satisfies StableContainedMaterialization; - }), - ); - yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); - const canonicalAfter = yield* fs.realPath(target); - if (!isContainedPath(path, repositoryRoot, canonicalAfter)) { - return yield* Effect.fail(new Error(`Repository file escaped its root while reading: ${relative}`)); - } - return materialized; - }).pipe(Effect.mapError(cause => new Error(`Could not safely materialize repository path ${relative}.`, {cause}))); -} - -const validateRepositoryAncestors = Effect.fn('codeGraph.validateRepositoryAncestors')(function* ( - fs: FileSystem.FileSystem, - path: Path.Path, - repositoryRoot: string, - relative: string, -) { - let current = repositoryRoot; - for (const segment of relative.split('/').slice(0, -1)) { - current = path.join(current, segment); - const link = yield* fs.readLink(current).pipe( - Effect.map(Option.some), - Effect.catch(() => Effect.succeed(Option.none())), - ); - if (Option.isSome(link)) { - return yield* Effect.fail(new Error(`Repository path has a symbolic ancestor: ${relative}`)); - } - const canonical = yield* fs.realPath(current); - const info = yield* fs.stat(current); - if (info.type !== 'Directory' || !isContainedPath(path, repositoryRoot, canonical)) { - return yield* Effect.fail(new Error(`Repository path has an unsafe ancestor: ${relative}`)); - } - } -}); - -function isContainedPath(path: Path.Path, root: string, candidate: string): boolean { - const relative = path.relative(root, candidate); - return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); -} - -function sameRegularFile( - before: FileSystem.File.Info, - current: FileSystem.File.Info, - opened: FileSystem.File.Info, -): boolean { - const beforeInode = Option.getOrUndefined(before.ino); - const currentInode = Option.getOrUndefined(current.ino); - const openedInode = Option.getOrUndefined(opened.ino); - return ( - before.type === 'File' && - current.type === 'File' && - opened.type === 'File' && - before.dev === current.dev && - current.dev === opened.dev && - beforeInode !== undefined && - currentInode !== undefined && - openedInode !== undefined && - beforeInode === currentInode && - currentInode === openedInode - ); -} - -function openedFilePath( - fs: FileSystem.FileSystem, - file: FileSystem.File, -): Effect.Effect, never, SystemInfo> { - const descriptor = (file as FileSystem.File & {readonly fd?: unknown}).fd; - if (typeof descriptor !== 'number' || !Number.isSafeInteger(descriptor) || descriptor < 0) { - return Effect.succeed(Option.none()); - } - return Effect.gen(function* () { - const system = yield* SystemInfo; - const descriptorPath = - system.platform === 'linux' - ? `/proc/self/fd/${descriptor}` - : system.platform === 'darwin' - ? `/dev/fd/${descriptor}` - : undefined; - if (!descriptorPath) return Option.none(); - const resolved = yield* fs.realPath(descriptorPath).pipe(Effect.option); - return Option.isSome(resolved) && resolved.value !== descriptorPath - ? Option.some(resolved.value) - : Option.none(); - }); -} diff --git a/src/code_graph/inventory_contained_file.ts b/src/code_graph/inventory_contained_file.ts new file mode 100644 index 00000000..31e85462 --- /dev/null +++ b/src/code_graph/inventory_contained_file.ts @@ -0,0 +1,392 @@ +import {Effect, FileSystem, Option, Path} from 'effect'; +import {SystemInfo} from '../effect/system.js'; +import {decodeUtf8} from './inventory_content.js'; +import {CodeGraphInventoryError} from './inventory_error.js'; + +export const readOptionalText = Effect.fn('codeGraph.readOptionalText')(function* ( + fs: FileSystem.FileSystem, + target: string, +) { + const opened = yield* readStableRegularFile(fs, target).pipe(Effect.option); + return opened._tag === 'Some' ? (decodeUtf8(opened.value.bytes) ?? '') : ''; +}); + +interface StableRegularFile { + readonly bytes: Uint8Array; + readonly identity: FileSystem.File.Info; + readonly openedPath: Option.Option; +} + +export interface ContainedReadInterlock { + readonly afterOpen?: Effect.Effect; + readonly beforeOpen?: Effect.Effect; +} + +function readStableRegularFile( + fs: FileSystem.FileSystem, + target: string, + interlock?: ContainedReadInterlock, +): Effect.Effect { + return Effect.gen(function* () { + const linkTarget = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + if (Option.isSome(linkTarget)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Refusing to read a symbolic repository file: ${target}`)); + } + const pathInfoBefore = yield* fs.stat(target); + if (pathInfoBefore.type !== 'File') { + return yield* Effect.fail( + new CodeGraphInventoryError(`Refusing to read a non-regular repository file: ${target}`), + ); + } + yield* interlock?.beforeOpen ?? Effect.void; + return yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(target, {flag: 'r'}); + yield* interlock?.afterOpen ?? Effect.void; + const openedInfoBefore = yield* file.stat; + const openedPath = yield* openedFilePath(fs, file); + const pathInfoOpened = yield* fs.stat(target); + if (!sameRegularFile(pathInfoBefore, pathInfoOpened, openedInfoBefore)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file changed while it was opened: ${target}`), + ); + } + const byteLength = Number(openedInfoBefore.size); + if (!Number.isSafeInteger(byteLength) || byteLength < 0) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file size cannot be represented safely: ${target}`), + ); + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + while (offset < bytes.byteLength) { + const read = Number(yield* file.read(bytes.subarray(offset))); + if (read <= 0) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file ended while it was being read: ${target}`), + ); + } + offset += read; + } + const openedInfoAfter = yield* file.stat; + const linkTargetAfter = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + if (Option.isSome(linkTargetAfter)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file became a symbolic link while reading: ${target}`), + ); + } + const pathInfoAfter = yield* fs.stat(target); + if ( + !sameRegularFile(pathInfoBefore, pathInfoAfter, openedInfoAfter) || + openedInfoBefore.size !== openedInfoAfter.size + ) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file changed while it was being read: ${target}`), + ); + } + return {bytes, identity: openedInfoAfter, openedPath}; + }), + ); + }).pipe( + Effect.mapError(cause => new CodeGraphInventoryError(`Could not safely read repository file ${target}.`, {cause})), + ); +} + +export function readContainedStableRegularFile( + fs: FileSystem.FileSystem, + path: Path.Path, + repositoryRoot: string, + relative: string, + interlock?: ContainedReadInterlock, +): Effect.Effect { + const target = path.join(repositoryRoot, ...relative.split('/')); + return Effect.gen(function* () { + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalBefore = yield* fs.realPath(target); + if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Repository file resolves outside its root: ${relative}`)); + } + const opened = yield* readStableRegularFile(fs, target, interlock); + if (Option.isSome(opened.openedPath) && !isContainedPath(path, repositoryRoot, opened.openedPath.value)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Opened repository file is outside its root: ${relative}`)); + } + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalAfter = yield* fs.realPath(target); + const finalInfo = yield* fs.stat(target); + if (!isContainedPath(path, repositoryRoot, canonicalAfter)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file escaped its root while reading: ${relative}`), + ); + } + if (!sameRegularFile(opened.identity, finalInfo, opened.identity)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository path no longer identifies the opened file: ${relative}`), + ); + } + return opened.bytes; + }).pipe( + Effect.mapError( + cause => new CodeGraphInventoryError(`Could not safely read repository path ${relative}.`, {cause}), + ), + ); +} + +interface StableContainedMaterialization { + readonly bytes?: Uint8Array; + readonly contentHash: string; + readonly size: number; +} + +export interface StableContainedRegularFileMetadata { + readonly size: number; +} + +/** + * Inspect only stable, contained path metadata. Admission depends on path and + * size, so excluded dirty files never need to be opened, read, or hashed. + */ +export function inspectContainedStableRegularFile( + fs: FileSystem.FileSystem, + path: Path.Path, + repositoryRoot: string, + relative: string, +): Effect.Effect { + const target = path.join(repositoryRoot, ...relative.split('/')); + return Effect.gen(function* () { + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalBefore = yield* fs.realPath(target); + if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Repository file resolves outside its root: ${relative}`)); + } + const linkBefore = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + if (Option.isSome(linkBefore)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Refusing to inspect a symbolic repository file: ${relative}`), + ); + } + const infoBefore = yield* fs.stat(target); + const size = Number(infoBefore.size); + if (infoBefore.type !== 'File' || !Number.isSafeInteger(size) || size < 0) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file metadata is not safely representable: ${relative}`), + ); + } + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalAfter = yield* fs.realPath(target); + const linkAfter = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + const infoAfter = yield* fs.stat(target); + if ( + Option.isSome(linkAfter) || + !isContainedPath(path, repositoryRoot, canonicalAfter) || + !sameRegularFile(infoBefore, infoAfter, infoBefore) || + infoBefore.size !== infoAfter.size + ) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file changed while its metadata was inspected: ${relative}`), + ); + } + return {size}; + }).pipe( + Effect.mapError( + cause => new CodeGraphInventoryError(`Could not safely inspect repository path ${relative}.`, {cause}), + ), + ); +} + +/** + * Safely materialize a worktree file, or hash it through a fixed-size buffer when + * policy says its content should remain metadata-only. This keeps dirty large + * corpus artifacts from allocating their full size while preserving an exact + * content fingerprint and the same symlink/race interlocks as ordinary reads. + */ +export function materializeContainedStableRegularFile( + fs: FileSystem.FileSystem, + path: Path.Path, + repositoryRoot: string, + relative: string, + omitContent: (size: number) => boolean, + expectedSize?: number, +): Effect.Effect { + const target = path.join(repositoryRoot, ...relative.split('/')); + return Effect.gen(function* () { + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalBefore = yield* fs.realPath(target); + if (!isContainedPath(path, repositoryRoot, canonicalBefore)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Repository file resolves outside its root: ${relative}`)); + } + const linkTarget = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + if (Option.isSome(linkTarget)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Refusing to read a symbolic repository file: ${relative}`), + ); + } + const pathInfoBefore = yield* fs.stat(target); + if (pathInfoBefore.type !== 'File') { + return yield* Effect.fail( + new CodeGraphInventoryError(`Refusing to read a non-regular repository file: ${relative}`), + ); + } + const materialized = yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(target, {flag: 'r'}); + const openedInfoBefore = yield* file.stat; + const openedPath = yield* openedFilePath(fs, file); + const pathInfoOpened = yield* fs.stat(target); + if (!sameRegularFile(pathInfoBefore, pathInfoOpened, openedInfoBefore)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file changed while it was opened: ${relative}`), + ); + } + if (Option.isSome(openedPath) && !isContainedPath(path, repositoryRoot, openedPath.value)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Opened repository file is outside its root: ${relative}`), + ); + } + const size = Number(openedInfoBefore.size); + if (!Number.isSafeInteger(size) || size < 0) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file size cannot be represented safely: ${relative}`), + ); + } + if (expectedSize !== undefined && size !== expectedSize) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file size changed before it was read: ${relative}`), + ); + } + const hasher = new Bun.CryptoHasher('sha256'); + const bytes = omitContent(size) ? undefined : new Uint8Array(size); + const buffer = bytes ?? new Uint8Array(Math.min(1_048_576, Math.max(1, size))); + let offset = 0; + while (offset < size) { + const view = bytes ? bytes.subarray(offset) : buffer.subarray(0, Math.min(buffer.byteLength, size - offset)); + const read = Number(yield* file.read(view)); + if (read <= 0) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file ended while it was being read: ${relative}`), + ); + } + hasher.update(view.subarray(0, read)); + offset += read; + } + const openedInfoAfter = yield* file.stat; + const linkTargetAfter = yield* fs.readLink(target).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + const pathInfoAfter = yield* fs.stat(target); + if ( + Option.isSome(linkTargetAfter) || + !sameRegularFile(pathInfoBefore, pathInfoAfter, openedInfoAfter) || + openedInfoBefore.size !== openedInfoAfter.size + ) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file changed while it was being read: ${relative}`), + ); + } + return {bytes, contentHash: hasher.digest('hex'), size} satisfies StableContainedMaterialization; + }), + ); + yield* validateRepositoryAncestors(fs, path, repositoryRoot, relative); + const canonicalAfter = yield* fs.realPath(target); + if (!isContainedPath(path, repositoryRoot, canonicalAfter)) { + return yield* Effect.fail( + new CodeGraphInventoryError(`Repository file escaped its root while reading: ${relative}`), + ); + } + return materialized; + }).pipe( + Effect.mapError( + cause => new CodeGraphInventoryError(`Could not safely materialize repository path ${relative}.`, {cause}), + ), + ); +} + +const validateRepositoryAncestors = Effect.fn('codeGraph.validateRepositoryAncestors')(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + repositoryRoot: string, + relative: string, +) { + let current = repositoryRoot; + for (const segment of relative.split('/').slice(0, -1)) { + current = path.join(current, segment); + const link = yield* fs.readLink(current).pipe( + Effect.map(Option.some), + Effect.catch(() => Effect.succeed(Option.none())), + ); + if (Option.isSome(link)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Repository path has a symbolic ancestor: ${relative}`)); + } + const canonical = yield* fs.realPath(current); + const info = yield* fs.stat(current); + if (info.type !== 'Directory' || !isContainedPath(path, repositoryRoot, canonical)) { + return yield* Effect.fail(new CodeGraphInventoryError(`Repository path has an unsafe ancestor: ${relative}`)); + } + } +}); + +function isContainedPath(path: Path.Path, root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +} + +function sameRegularFile( + before: FileSystem.File.Info, + current: FileSystem.File.Info, + opened: FileSystem.File.Info, +): boolean { + const beforeInode = Option.getOrUndefined(before.ino); + const currentInode = Option.getOrUndefined(current.ino); + const openedInode = Option.getOrUndefined(opened.ino); + return ( + before.type === 'File' && + current.type === 'File' && + opened.type === 'File' && + before.dev === current.dev && + current.dev === opened.dev && + beforeInode !== undefined && + currentInode !== undefined && + openedInode !== undefined && + beforeInode === currentInode && + currentInode === openedInode + ); +} + +function openedFilePath( + fs: FileSystem.FileSystem, + file: FileSystem.File, +): Effect.Effect, never, SystemInfo> { + const descriptor = (file as FileSystem.File & {readonly fd?: unknown}).fd; + if (typeof descriptor !== 'number' || !Number.isSafeInteger(descriptor) || descriptor < 0) { + return Effect.succeed(Option.none()); + } + return Effect.gen(function* () { + const system = yield* SystemInfo; + const descriptorPath = + system.platform === 'linux' + ? `/proc/self/fd/${descriptor}` + : system.platform === 'darwin' + ? `/dev/fd/${descriptor}` + : undefined; + if (!descriptorPath) return Option.none(); + const resolved = yield* fs.realPath(descriptorPath).pipe(Effect.option); + return Option.isSome(resolved) && resolved.value !== descriptorPath + ? Option.some(resolved.value) + : Option.none(); + }); +} diff --git a/src/code_graph/inventory_content.ts b/src/code_graph/inventory_content.ts new file mode 100644 index 00000000..8262a56f --- /dev/null +++ b/src/code_graph/inventory_content.ts @@ -0,0 +1,306 @@ +import {Option} from 'effect'; +import {BUILTIN_LANGUAGE_PACK_REGISTRY, type CodeGraphLanguagePackRegistryShape} from './languages/registry.js'; +import {CORPUS_EXTRACTION_SOURCE_BYTES_LIMIT} from './languages/corpus/policy.js'; +import {isLowSignalStructuredPath} from './languages/schemas/policy.js'; +import type {CodeGraphInventoryFile} from './types.js'; + +const COMPACT_RESOLUTION_CONTEXT_NAMES = new Set([ + 'build.gradle', + 'build.gradle.kts', + 'go.mod', + 'gradle.properties', + 'package.json', + 'package.swift', + 'pom.xml', + 'project.pbxproj', + 'settings.gradle', + 'settings.gradle.kts', + 'tsconfig.json', +]); + +export function retainResolutionContext( + file: CodeGraphInventoryFile, + languagePacks: CodeGraphLanguagePackRegistryShape, +): CodeGraphInventoryFile { + const name = file.path.split('/').at(-1)?.toLowerCase() ?? ''; + const content = + file.content === undefined + ? undefined + : (compactResolutionContext(name, file.content) ?? + (languagePacks.isResolutionContext(file.path) && !COMPACT_RESOLUTION_CONTEXT_NAMES.has(name) + ? file.content + : undefined)); + if (content !== undefined) { + return {...file, content}; + } + const {bytes: _bytes, content: _content, contentOmittedReason: _contentOmittedReason, ...metadata} = file; + return metadata; +} + +function isCorpusContent(path: string, languagePacks: CodeGraphLanguagePackRegistryShape): boolean { + return Option.match(languagePacks.match(path), { + onNone: () => false, + onSome: value => value.role === 'corpus', + }); +} + +export function shouldOmitRepositoryContent( + path: string, + size: number, + languagePacks: CodeGraphLanguagePackRegistryShape = BUILTIN_LANGUAGE_PACK_REGISTRY, +): boolean { + return repositoryContentOmissionReason(path, size, languagePacks) !== undefined; +} + +export function repositoryContentOmissionReason( + path: string, + size: number, + languagePacks: CodeGraphLanguagePackRegistryShape, +): CodeGraphInventoryFile['contentOmittedReason'] { + const match = languagePacks.match(path); + if (Option.isNone(match)) return undefined; + if ( + (match.value.language === 'json' || match.value.language === 'jsonc' || match.value.language === 'yaml') && + isLowSignalStructuredPath(path) + ) { + return 'metadata-only'; + } + if (match.value.role !== 'corpus') return undefined; + if (size > CORPUS_EXTRACTION_SOURCE_BYTES_LIMIT) return 'size-budget'; + return match.value.language === 'image' || match.value.language === 'audio' || match.value.language === 'video' + ? 'metadata-only' + : undefined; +} + +export function acceptsBinaryContent(path: string, languagePacks: CodeGraphLanguagePackRegistryShape): boolean { + return isCorpusContent(path, languagePacks); +} + +function compactResolutionContext(name: string, content: string): string | undefined { + if (name === 'go.mod') return compactGoModule(content); + if (name === 'pom.xml') return compactMavenManifest(content); + if (name === 'settings.gradle' || name === 'settings.gradle.kts') return compactGradleSettings(content); + if (name === 'build.gradle' || name === 'build.gradle.kts') return compactGradleBuild(content); + if (name === 'gradle.properties') return ''; + if (name === 'package.swift') return compactSwiftPackage(content); + if (name === 'project.pbxproj') return compactXcodeProject(content); + if (name !== 'package.json' && name !== 'tsconfig.json') return undefined; + try { + const parsed = JSON.parse(content) as Record; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined; + if (name === 'package.json') { + const entry = packageEntryForResolution(parsed.exports, parsed.main); + const dependencySections = Object.fromEntries( + ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies'].flatMap(section => { + const value = parsed[section]; + if (!value || typeof value !== 'object' || Array.isArray(value)) return []; + return [ + [ + section, + Object.fromEntries( + Object.entries(value as Record).flatMap(([dependency, version]) => + typeof version === 'string' ? [[dependency, version]] : [], + ), + ), + ], + ]; + }), + ); + return JSON.stringify({ + ...dependencySections, + ...(entry === undefined ? {} : {main: entry}), + ...(typeof parsed.name === 'string' ? {name: parsed.name} : {}), + ...(typeof parsed.packageManager === 'string' ? {packageManager: parsed.packageManager} : {}), + ...(Array.isArray(parsed.workspaces) + ? {workspaces: parsed.workspaces.filter((value): value is string => typeof value === 'string')} + : parsed.workspaces && typeof parsed.workspaces === 'object' + ? { + workspaces: { + packages: Array.isArray((parsed.workspaces as Record).packages) + ? ((parsed.workspaces as Record).packages as unknown[]).filter( + (value): value is string => typeof value === 'string', + ) + : [], + }, + } + : {}), + }); + } + const compilerOptions = + parsed.compilerOptions && typeof parsed.compilerOptions === 'object' && !Array.isArray(parsed.compilerOptions) + ? (parsed.compilerOptions as Record) + : {}; + const paths = + compilerOptions.paths && typeof compilerOptions.paths === 'object' && !Array.isArray(compilerOptions.paths) + ? Object.fromEntries( + Object.entries(compilerOptions.paths as Record).flatMap(([alias, targets]) => + Array.isArray(targets) + ? [[alias, targets.filter((target): target is string => typeof target === 'string')]] + : [], + ), + ) + : undefined; + const compact: Record = { + compilerOptions: { + ...(typeof compilerOptions.baseUrl === 'string' ? {baseUrl: compilerOptions.baseUrl} : {}), + ...(typeof compilerOptions.outDir === 'string' ? {outDir: compilerOptions.outDir} : {}), + ...(paths === undefined ? {} : {paths}), + }, + }; + for (const field of ['exclude', 'files', 'include'] as const) { + if (Object.prototype.hasOwnProperty.call(parsed, field)) { + compact[field] = Array.isArray(parsed[field]) + ? parsed[field].filter((value): value is string => typeof value === 'string') + : parsed[field]; + } + } + if (Array.isArray(parsed.references)) { + compact.references = parsed.references.flatMap(reference => + reference && + typeof reference === 'object' && + !Array.isArray(reference) && + typeof (reference as Record).path === 'string' + ? [{path: (reference as Record).path}] + : [], + ); + } + return JSON.stringify(compact); + } catch { + return undefined; + } +} + +function compactGoModule(content: string): string { + const output: string[] = []; + let inRequireBlock = false; + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.replace(/\/\/.*$/, '').trim(); + if (!line) continue; + if (/^module\s+\S+/.test(line)) { + output.push(line); + continue; + } + if (/^require\s*\($/.test(line)) { + inRequireBlock = true; + continue; + } + if (inRequireBlock && line === ')') { + inRequireBlock = false; + continue; + } + if (inRequireBlock && /^\S+\s+v\S+/.test(line)) { + output.push(line); + continue; + } + if (/^require\s+\S+\s+v\S+/.test(line)) output.push(line); + } + return `${output.join('\n')}\n`; +} + +function compactMavenManifest(content: string): string | undefined { + const project = content.replace(//i, ''); + const group = compactXmlTag(project, 'groupId'); + const artifact = compactXmlTag(project, 'artifactId'); + if (!artifact) return undefined; + const modules = compactXmlTags(content, 'module'); + const dependencies = [...content.matchAll(//gi)].flatMap(match => { + const dependencyArtifact = compactXmlTag(match[0], 'artifactId'); + if (!dependencyArtifact) return []; + const dependencyGroup = compactXmlTag(match[0], 'groupId'); + return [ + `${dependencyGroup ? `${dependencyGroup}` : ''}${dependencyArtifact}`, + ]; + }); + return [ + '', + group ? `${group}` : '', + `${artifact}`, + modules.length > 0 ? `${modules.map(module => `${module}`).join('')}` : '', + dependencies.length > 0 ? `${dependencies.join('')}` : '', + '', + ].join(''); +} + +function compactGradleSettings(content: string): string { + return `${content + .split(/\r?\n/) + .filter(line => /\brootProject\.name\b|^\s*include\b|\.projectDir\s*=/.test(line)) + .join('\n')}\n`; +} + +function compactGradleBuild(content: string): string { + return `${content + .split(/\r?\n/) + .filter(line => /\bproject\s*\(/.test(line)) + .join('\n')}\n`; +} + +function compactSwiftPackage(content: string): string | undefined { + const packageName = /\bPackage\s*\(\s*name\s*:\s*"([^"]+)"/m.exec(content)?.[1]; + const starts = [...content.matchAll(/\.(target|executableTarget|testTarget)\s*\(\s*name\s*:\s*"([^"]+)"/g)]; + if (!packageName && starts.length === 0) return undefined; + const targets = starts.map((match, index) => { + const body = content.slice(match.index, starts[index + 1]?.index ?? content.length); + const path = /\bpath\s*:\s*"([^"]+)"/.exec(body)?.[1]; + const dependencies = /\bdependencies\s*:\s*\[([\s\S]*?)\]/.exec(body)?.[1] ?? ''; + const names = [...dependencies.matchAll(/"([^"]+)"/g)].map(value => value[1]!); + return `.${match[1]}(name: ${JSON.stringify(match[2])}, dependencies: [${names + .map(name => JSON.stringify(name)) + .join(', ')}]${path ? `, path: ${JSON.stringify(path)}` : ''})`; + }); + return `let package = Package(name: ${JSON.stringify(packageName ?? 'Package')}, targets: [${targets.join(', ')}])\n`; +} + +function compactXcodeProject(content: string): string { + const targets = [...content.matchAll(/isa\s*=\s*PBXNativeTarget;[\s\S]*?\bname\s*=\s*"?([^";\n]+)"?;/g)].map(match => + match[1]!.trim(), + ); + return `${targets.map(target => `isa = PBXNativeTarget; name = ${JSON.stringify(target)};`).join('\n')}\n`; +} + +function compactXmlTag(content: string, tag: string): string | undefined { + return new RegExp(`<${tag}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*`, 'i').exec(content)?.[1]?.trim(); +} + +function compactXmlTags(content: string, tag: string): readonly string[] { + return [...content.matchAll(new RegExp(`<${tag}(?:\\s[^>]*)?>\\s*([^<]+?)\\s*`, 'gi'))].map(match => + match[1]!.trim(), + ); +} + +function packageEntryForResolution(exportsValue: unknown, mainValue: unknown): string | undefined { + if (exportsValue === undefined) return typeof mainValue === 'string' ? mainValue : undefined; + const root = + typeof exportsValue === 'object' && + exportsValue !== null && + !Array.isArray(exportsValue) && + Object.keys(exportsValue).some(key => key.startsWith('.')) + ? (exportsValue as Record)['.'] + : exportsValue; + const targets = new Set(collectResolutionExportTargets(root)); + return targets.size === 1 ? [...targets][0] : undefined; +} + +function collectResolutionExportTargets(value: unknown): readonly string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(collectResolutionExportTargets); + if (typeof value !== 'object' || value === null) return []; + return Object.values(value as Record).flatMap(collectResolutionExportTargets); +} + +export function appearsBinary(bytes: Uint8Array): boolean { + return bytes.subarray(0, Math.min(bytes.byteLength, 8192)).includes(0); +} + +export function appearsGitLfsPointer(bytes: Uint8Array): boolean { + if (bytes.byteLength > 1024) return false; + return new TextDecoder().decode(bytes).startsWith('version https://git-lfs.github.com/spec/v1\n'); +} + +export function decodeUtf8(bytes: Uint8Array): string | undefined { + try { + return new TextDecoder('utf-8', {fatal: true}).decode(bytes); + } catch { + return undefined; + } +} diff --git a/src/code_graph/inventory_error.ts b/src/code_graph/inventory_error.ts new file mode 100644 index 00000000..03dfaced --- /dev/null +++ b/src/code_graph/inventory_error.ts @@ -0,0 +1,3 @@ +export class CodeGraphInventoryError extends Error { + readonly _tag = 'CodeGraphInventoryError' as const; +} diff --git a/src/code_graph/isolated_builder.ts b/src/code_graph/isolated_builder.ts index 3e5c4ae5..490b9d39 100644 --- a/src/code_graph/isolated_builder.ts +++ b/src/code_graph/isolated_builder.ts @@ -1,5 +1,5 @@ import {Clock, Effect, Option, Path, Ref} from 'effect'; -import {fromPromiseError} from '../effect/errors.js'; +import {fromPromiseInterruptible} from '../effect/errors.js'; import {SystemInfo, type SystemInfoShape} from '../effect/system.js'; import {pollUntilEffect} from '../effect/time.js'; @@ -14,6 +14,17 @@ import {codeGraphLayout} from './layout.js'; import {resolveRepositoryIdentity} from './repository.js'; import type {CodeGraphProgress, RepositoryIdentity} from './types.js'; +class IsolatedBuilderError extends Error { + readonly _tag = 'IsolatedBuilderError' as const; +} + +const isolatedBuilderPromise = (operation: string, evaluate: () => PromiseLike) => + fromPromiseInterruptible( + evaluate, + cause => + new IsolatedBuilderError(`${operation}: ${cause instanceof Error ? cause.message : String(cause)}`, {cause}), + ); + /** Match the child heartbeat cadence so MCP does not oversample process-liveness probes. */ export const BUILD_STATUS_POLL_MILLISECONDS = CODE_GRAPH_BUILD_HEARTBEAT_INTERVAL_MILLISECONDS; /** Give up awaiting a wedged foreign builder after this much continuous stall. */ @@ -209,7 +220,7 @@ export function codeGraphProgressFromBuildStatus( }; default: { const _exhaustive: never = status.phase; - throw new Error(`Unsupported code graph progress phase: ${String(_exhaustive)}`); + throw new IsolatedBuilderError(`Unsupported code graph progress phase: ${String(_exhaustive)}`); } } } @@ -241,11 +252,13 @@ export const runIsolatedCodeGraphIndex = Effect.fn('codeGraph.isolatedBuilder.ru const priorBuildId = existing?.buildId; const spawn = options.spawn ?? spawnIsolatedBuilderProcess; // Detach on interruption: do not kill multi-hour builds when the MCP host goes idle or reconnects. - const child = yield* fromPromiseError(() => Promise.resolve(spawn(plan))); + const child = yield* isolatedBuilderPromise('Could not spawn isolated code graph builder', () => + Promise.resolve(spawn(plan)), + ); const observedBuildId = yield* Ref.make(undefined); const exitCode = yield* Effect.raceFirst( - fromPromiseError(() => child.exited), + isolatedBuilderPromise('Could not await isolated code graph builder', () => child.exited), mirrorBuildStatusProgress(readStatus, child.processId, priorBuildId, observedBuildId, options.onProgress), ); @@ -253,7 +266,7 @@ export const runIsolatedCodeGraphIndex = Effect.fn('codeGraph.isolatedBuilder.ru // A failed child is never rescued by a later sidecar; only enrich its failure with the exact owned status. const failed = yield* statusOwnedBy(readStatus, child.processId, priorBuildId, yield* Ref.get(observedBuildId)); return yield* Effect.fail( - new Error(isolatedBuilderFailureMessage(exitCode, failed?.error?.summary, child.stderrTail?.())), + new IsolatedBuilderError(isolatedBuilderFailureMessage(exitCode, failed?.error?.summary, child.stderrTail?.())), ); } @@ -287,7 +300,7 @@ export function isolatedBuilderResultFromCompletedStatus( symbols: status.result.symbols, }); } - return Effect.fail(new Error('isolated graph index finished without writing a build result')); + return Effect.fail(new IsolatedBuilderError('isolated graph index finished without writing a build result')); } /** @internal Bounded completion-sidecar grace used after a successful isolated child exit. */ @@ -351,21 +364,24 @@ export function statusBelongsToChild( export function assertIsolatedBuilderPlan(plan: CodeGraphIsolatedBuilderSpawnPlan): void { const executableName = executableBaseName(plan.executable); if (executableName?.startsWith('threadnote-mcp-server') === true) { - throw new Error('Isolated graph builder must not spawn the MCP launcher executable.'); + throw new IsolatedBuilderError('Isolated graph builder must not spawn the MCP launcher executable.'); } const graphAt = plan.arguments.indexOf('graph'); if (graphAt < 0 || plan.arguments[graphAt + 1] !== 'index') { - throw new Error('Isolated graph builder spawn plan must invoke `graph index`.'); + throw new IsolatedBuilderError('Isolated graph builder spawn plan must invoke `graph index`.'); } if (plan.arguments.slice(0, graphAt).includes('mcp-server')) { - throw new Error('Isolated graph builder must not spawn an MCP server child.'); + throw new IsolatedBuilderError('Isolated graph builder must not spawn an MCP server child.'); } } function assertIsolatedBuilderPlanEffect(plan: CodeGraphIsolatedBuilderSpawnPlan) { return Effect.try({ try: () => assertIsolatedBuilderPlan(plan), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => + cause instanceof IsolatedBuilderError + ? cause + : new IsolatedBuilderError(cause instanceof Error ? cause.message : String(cause), {cause}), }); } @@ -440,7 +456,9 @@ function awaitExistingBuilder( for (;;) { const status = yield* readStatus; if (!status) { - return yield* Effect.fail(new Error('Existing code graph builder status disappeared before completion.')); + return yield* Effect.fail( + new IsolatedBuilderError('Existing code graph builder status disappeared before completion.'), + ); } if (status.observation.liveness === 'completed' && status.result) { return { @@ -449,11 +467,13 @@ function awaitExistingBuilder( } satisfies CodeGraphIsolatedBuilderResult; } if (status.observation.liveness === 'completed') { - return yield* Effect.fail(new Error('Existing code graph builder completed without writing a build result.')); + return yield* Effect.fail( + new IsolatedBuilderError('Existing code graph builder completed without writing a build result.'), + ); } if (status.observation.liveness !== 'active' && status.observation.liveness !== 'stalled') { return yield* Effect.fail( - new Error(status.error?.summary ?? 'Existing code graph builder stopped before completion.'), + new IsolatedBuilderError(status.error?.summary ?? 'Existing code graph builder stopped before completion.'), ); } if (status.observation.liveness === 'stalled') { @@ -464,7 +484,7 @@ function awaitExistingBuilder( }); if (now - started >= EXISTING_BUILDER_STALLED_TIMEOUT_MILLISECONDS) { return yield* Effect.fail( - new Error('Existing code graph builder stalled without progress; retry the refresh.'), + new IsolatedBuilderError('Existing code graph builder stalled without progress; retry the refresh.'), ); } } else { @@ -504,7 +524,7 @@ function mirrorBuildStatusProgress( } function emitProgress(onProgress: CodeGraphIsolatedBuilderOptions['onProgress'], progress: CodeGraphProgress) { - return (onProgress?.(progress) ?? Effect.void).pipe(Effect.catch(() => Effect.void)); + return onProgress?.(progress) ?? Effect.void; } function developmentStandaloneScript(system: SystemInfoShape): Option.Option { diff --git a/src/code_graph/lifecycle_opportunity.ts b/src/code_graph/lifecycle_opportunity.ts index a1736dc7..ce0a5368 100644 --- a/src/code_graph/lifecycle_opportunity.ts +++ b/src/code_graph/lifecycle_opportunity.ts @@ -75,7 +75,7 @@ export const observeCodeGraphLifecycleOpportunityTargets = Effect.fn('codeGraph. checkoutId, repositoryId: view.repositoryId, worktreeId: view.worktreeId, - }).pipe(Effect.catch(() => Effect.succeed({available: false, state: 'invalid'} as const))), + }), {concurrency: 1}, ); const anchor = associations.find(association => association.state === 'verified' && 'path' in association); diff --git a/src/code_graph/local_provenance.ts b/src/code_graph/local_provenance.ts index 13b2a470..2d3f7e29 100644 --- a/src/code_graph/local_provenance.ts +++ b/src/code_graph/local_provenance.ts @@ -10,10 +10,14 @@ import { type CodeGraphGitWorktreeRegistration, } from './git_worktree_registration.js'; import {classifyCodeGraphLifecycle} from './lifecycle_classification.js'; -import {resolveRepositoryIdentityDetail} from './repository.js'; +import {normalizeRepositoryBranchName, resolveRepositoryIdentityDetail} from './repository.js'; import {codeGraphLocalProvenanceLockPath} from './layout.js'; import type {RepositoryIdentity} from './types.js'; +class CodeGraphLocalProvenanceError extends Error { + readonly _tag = 'CodeGraphLocalProvenanceError' as const; +} + const LOCAL_CONTEXT_DIRECTORY = 'local-context'; const LOCAL_WORKTREES_DIRECTORY = 'worktrees'; const LOCAL_PROVENANCE_LEGACY_SCHEMA_VERSION = 1 as const; @@ -28,6 +32,7 @@ export type CodeGraphLocalAssociationState = 'invalid' | 'legacy-unknown' | 'mis export interface CodeGraphLocalAssociation { readonly available: boolean; + readonly branch?: string; /** Home-abbreviated path for trusted local human interfaces. */ readonly displayPath?: string; readonly observedAt?: string; @@ -43,6 +48,7 @@ export interface CodeGraphLocalAssociationTarget { } interface CodeGraphLocalProvenanceRecordBase { + readonly branch?: string; readonly canonicalWorktreePath: string; readonly checkoutId: string; readonly headCommit?: string; @@ -244,6 +250,7 @@ const recordResolvedCodeGraphLocalAssociationUnlocked = Effect.fn('codeGraph.rec if ( existing?.canonicalWorktreePath === identity.repoRoot && existing.headCommit === identity.headCommit && + existing.branch === identity.branch && existing.schemaVersion === LOCAL_PROVENANCE_SCHEMA_VERSION && sameCodeGraphGitWorktreeRegistration(existing.registration, registration.value) && now - Date.parse(existing.observedAt) >= 0 && @@ -253,6 +260,7 @@ const recordResolvedCodeGraphLocalAssociationUnlocked = Effect.fn('codeGraph.rec } const record = { + ...(identity.branch === undefined ? {} : {branch: identity.branch}), canonicalWorktreePath: identity.repoRoot, checkoutId: identity.checkoutId, headCommit: identity.headCommit, @@ -282,10 +290,14 @@ const recordResolvedCodeGraphLocalAssociationUnlocked = Effect.fn('codeGraph.rec directory.value, ); if (revalidated !== directory.value) { - return yield* Effect.fail(new Error('Code graph local provenance directory changed during observation.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory changed during observation.'), + ); } if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance target is a symbolic link.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance target is a symbolic link.'), + ); } yield* options.beforePublishValidation?.() ?? Effect.void; const finalIdentity = yield* resolveMatchingRepositoryIdentity(identity); @@ -294,7 +306,9 @@ const recordResolvedCodeGraphLocalAssociationUnlocked = Effect.fn('codeGraph.rec finalIdentity.gitDirectory, ); if (!sameCodeGraphGitWorktreeRegistration(record.registration, finalRegistration)) { - return yield* Effect.fail(new Error('Code graph local provenance registration changed during observation.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance registration changed during observation.'), + ); } yield* fs.rename(temporary, target); yield* syncDirectory(fs, directory.value); @@ -325,7 +339,9 @@ export const readCodeGraphLocalAssociation = Effect.fn('codeGraph.readLocalAssoc identity.worktreeId === target.worktreeId && (target.repositoryId === undefined || identity.repositoryId === target.repositoryId) ? Effect.void - : Effect.fail(new Error('Live worktree identity does not match the requested graph association.')), + : Effect.fail( + new CodeGraphLocalProvenanceError('Live worktree identity does not match the requested graph association.'), + ), }).pipe(Effect.option); return Option.isSome(resolved) && resolved.value.association.state === 'verified' ? resolved.value.association @@ -620,16 +636,20 @@ const readLocalProvenanceCleanupCandidate = Effect.fn('codeGraph.readLocalProven Number(info.size) > LOCAL_PROVENANCE_BYTES_LIMIT || (system.platform !== 'win32' && (info.mode & 0o777) !== 0o600) ) { - return yield* Effect.fail(new Error('Code graph local provenance cleanup target is invalid.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance cleanup target is invalid.'), + ); } const content = yield* readBoundedObservedRegularFile(fs, file, info); if (Option.isSome(yield* fs.readLink(file).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance cleanup target changed.')); + return yield* Effect.fail(new CodeGraphLocalProvenanceError('Code graph local provenance cleanup target changed.')); } const record = parseCodeGraphLocalProvenanceRecordJson(content, target); if (record?.schemaVersion !== LOCAL_PROVENANCE_SCHEMA_VERSION) return undefined; if (!isCanonicalAbsolutePath(path, record.canonicalWorktreePath)) { - return yield* Effect.fail(new Error('Code graph local provenance cleanup record is invalid.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance cleanup record is invalid.'), + ); } const recordDigest = sha256HexSync(content); return { @@ -888,7 +908,8 @@ export function parseCodeGraphLocalProvenanceRecord( !isHash(value.repositoryId) || !isCanonicalTimestamp(value.observedAt) || !isLocalPath(value.canonicalWorktreePath) || - (value.headCommit !== undefined && (typeof value.headCommit !== 'string' || !COMMIT_ID.test(value.headCommit))) + (value.headCommit !== undefined && (typeof value.headCommit !== 'string' || !COMMIT_ID.test(value.headCommit))) || + (value.branch !== undefined && !isBranchName(value.branch)) ) { return undefined; } @@ -906,6 +927,7 @@ export function parseCodeGraphLocalProvenanceRecord( return undefined; } const base = { + ...(value.branch === undefined ? {} : {branch: value.branch}), canonicalWorktreePath: value.canonicalWorktreePath, checkoutId: value.checkoutId, ...(value.headCommit === undefined ? {} : {headCommit: value.headCommit}), @@ -939,7 +961,7 @@ function validTarget(target: CodeGraphLocalAssociationTarget): boolean { function resolveMatchingRepositoryIdentity(identity: RepositoryIdentity) { return Effect.gen(function* () { if (!validTarget(identity) || !COMMIT_ID.test(identity.headCommit)) { - return yield* Effect.fail(new Error('Code graph local provenance identity is invalid.')); + return yield* Effect.fail(new CodeGraphLocalProvenanceError('Code graph local provenance identity is invalid.')); } const resolvedDetail = yield* resolveRepositoryIdentityDetail(identity.repoRoot); const resolved = resolvedDetail.identity; @@ -949,9 +971,12 @@ function resolveMatchingRepositoryIdentity(identity: RepositoryIdentity) { resolved.checkoutId !== identity.checkoutId || resolved.worktreeId !== identity.worktreeId || resolved.repositoryId !== identity.repositoryId || - resolved.headCommit !== identity.headCommit + resolved.headCommit !== identity.headCommit || + resolved.branch !== identity.branch ) { - return yield* Effect.fail(new Error('Code graph local provenance identity changed before observation.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance identity changed before observation.'), + ); } return resolvedDetail; }); @@ -987,7 +1012,8 @@ function ensureLocalWorktreeDirectory( checkoutId: string, ) { return Effect.gen(function* () { - if (!HASH_ID.test(checkoutId)) return yield* Effect.fail(new Error('Code graph checkout identity is invalid.')); + if (!HASH_ID.test(checkoutId)) + return yield* Effect.fail(new CodeGraphLocalProvenanceError('Code graph checkout identity is invalid.')); const canonicalHome = yield* fs.realPath(threadnoteHome); const indexes = yield* ensureContainedDirectory(fs, path, canonicalHome, 'indexes', false); const codeGraph = yield* ensureContainedDirectory(fs, path, indexes, 'code-graph', false); @@ -1026,11 +1052,15 @@ function ensureContainedDirectory( return Effect.gen(function* () { const directory = path.join(canonicalParent, name); if (Option.isSome(yield* fs.readLink(directory).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance directory is a symbolic link.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory is a symbolic link.'), + ); } yield* fs.makeDirectory(directory, {recursive: true, mode: privateDirectory ? 0o700 : 0o755}); if (Option.isSome(yield* fs.readLink(directory).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance directory is a symbolic link.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory is a symbolic link.'), + ); } const canonical = yield* inspectContainedDirectory(fs, path, canonicalParent, directory); if (privateDirectory) yield* fs.chmod(canonical, 0o700); @@ -1047,7 +1077,9 @@ function inspectOptionalContainedDirectory( return Effect.gen(function* () { const directory = path.join(canonicalParent, name); if (Option.isSome(yield* fs.readLink(directory).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance directory is a symbolic link.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory is a symbolic link.'), + ); } if (!(yield* fs.exists(directory))) return undefined; return yield* inspectContainedDirectory(fs, path, canonicalParent, directory); @@ -1066,7 +1098,9 @@ function inspectPrivateContainedDirectory( const system = yield* SystemInfo; const mode = system.platform === 'win32' ? 0o700 : info.mode; if ((mode & 0o777) === 0o700) return canonical; - return yield* Effect.fail(new Error('Code graph local provenance directory permissions are not private.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory permissions are not private.'), + ); }); } @@ -1078,15 +1112,21 @@ function inspectContainedDirectory( ) { return Effect.gen(function* () { if (Option.isSome(yield* fs.readLink(directory).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph local provenance directory is a symbolic link.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance directory is a symbolic link.'), + ); } const info = yield* fs.stat(directory); if (info.type !== 'Directory') { - return yield* Effect.fail(new Error('Code graph local provenance path is not a directory.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance path is not a directory.'), + ); } const canonical = yield* fs.realPath(directory); if (path.dirname(canonical) !== canonicalParent || path.basename(canonical) !== path.basename(directory)) { - return yield* Effect.fail(new Error('Code graph local provenance path escaped its checkout.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance path escaped its checkout.'), + ); } return canonical; }); @@ -1097,6 +1137,7 @@ function associationForRecord(path: Path.Path, record: CodeGraphLocalProvenanceR const system = yield* SystemInfo; return { available: state === 'verified', + ...(record.branch === undefined ? {} : {branch: record.branch}), displayPath: homeAbbreviatedPath(path, system.homeDirectory, record.canonicalWorktreePath), observedAt: record.observedAt, path: record.canonicalWorktreePath, @@ -1124,6 +1165,10 @@ function isHash(value: unknown): value is string { return typeof value === 'string' && HASH_ID.test(value); } +function isBranchName(value: unknown): value is string { + return typeof value === 'string' && normalizeRepositoryBranchName(value) === value; +} + function isCanonicalTimestamp(value: unknown): value is string { if (typeof value !== 'string' || value.length > 40) return false; const milliseconds = Date.parse(value); @@ -1190,10 +1235,14 @@ function readBoundedObservedRegularFile(fs: FileSystem.FileSystem, file: string, !sameObservedRegularFile(pathInfoBefore, openedInfoBefore) || !sameObservedRegularFile(pathInfoBefore, pathInfoOpened) ) { - return yield* Effect.fail(new Error('Code graph local provenance changed while opening it.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance changed while opening it.'), + ); } if (Number(openedInfoBefore.size) > LOCAL_PROVENANCE_BYTES_LIMIT) { - return yield* Effect.fail(new Error('Code graph local provenance exceeds its bounded read limit.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance exceeds its bounded read limit.'), + ); } const bytes = new Uint8Array(LOCAL_PROVENANCE_BYTES_LIMIT + 1); @@ -1201,7 +1250,9 @@ function readBoundedObservedRegularFile(fs: FileSystem.FileSystem, file: string, while (offset < bytes.length) { const count = Number(yield* opened.read(bytes.subarray(offset))); if (!Number.isSafeInteger(count) || count < 0 || count > bytes.length - offset) { - return yield* Effect.fail(new Error('Code graph local provenance returned an invalid bounded read size.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance returned an invalid bounded read size.'), + ); } if (count === 0) break; offset += count; @@ -1213,14 +1264,18 @@ function readBoundedObservedRegularFile(fs: FileSystem.FileSystem, file: string, !sameObservedRegularFile(pathInfoBefore, openedInfoAfter) || !sameObservedRegularFile(pathInfoBefore, pathInfoAfter) ) { - return yield* Effect.fail(new Error('Code graph local provenance changed during its bounded read.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance changed during its bounded read.'), + ); } if (offset > LOCAL_PROVENANCE_BYTES_LIMIT || BigInt(offset) !== openedInfoBefore.size) { - return yield* Effect.fail(new Error('Code graph local provenance changed size during its bounded read.')); + return yield* Effect.fail( + new CodeGraphLocalProvenanceError('Code graph local provenance changed size during its bounded read.'), + ); } return yield* Effect.try({ try: () => new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}).decode(bytes.subarray(0, offset)), - catch: cause => new Error('Code graph local provenance is not valid UTF-8.', {cause}), + catch: cause => new CodeGraphLocalProvenanceError('Code graph local provenance is not valid UTF-8.', {cause}), }); }), ); diff --git a/src/code_graph/maintenance.ts b/src/code_graph/maintenance.ts index 7b59f438..eef9950b 100644 --- a/src/code_graph/maintenance.ts +++ b/src/code_graph/maintenance.ts @@ -22,6 +22,10 @@ import {diagnoseCodeGraphDatabase} from './deep_diagnostics.js'; export {diagnoseCodeGraphDatabaseReadOnly} from './store_health.js'; +class CodeGraphMaintenanceError extends Error { + readonly _tag = 'CodeGraphMaintenanceError' as const; +} + const CODE_GRAPH_EXPLICIT_SCHEMA_PREPARATION_STEP_LIMIT = 8; export interface CodeGraphRepairSummary { @@ -121,7 +125,7 @@ export const inspectObsoleteCodeGraphStores = Effect.fn('codeGraph.inspectObsole const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (checkoutId !== undefined && !/^[0-9a-f]{64}$/.test(checkoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity is invalid.')); + return yield* Effect.fail(new CodeGraphMaintenanceError('Code graph checkout identity is invalid.')); } const repositories = codeGraphRepositoriesRoot(path, threadnoteHome); if (!(yield* fs.exists(repositories))) return emptyObsoleteInventory(); @@ -281,7 +285,7 @@ export const repairCodeGraphIndexes = Effect.fn('codeGraph.repairIndexes')(funct const path = yield* Path.Path; const store = yield* CodeGraphStore; if (options.targetCheckoutId !== undefined && !/^[0-9a-f]{64}$/.test(options.targetCheckoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity is invalid.')); + return yield* Effect.fail(new CodeGraphMaintenanceError('Code graph checkout identity is invalid.')); } const repair = Effect.gen(function* () { const allDatabases = yield* codeGraphDatabasePaths(threadnoteHome); @@ -547,7 +551,7 @@ export const purgeObsoleteCodeGraphStores = Effect.fn('codeGraph.purgeObsoleteSt const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (!/^[0-9a-f]{64}$/.test(checkoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity is invalid.')); + return yield* Effect.fail(new CodeGraphMaintenanceError('Code graph checkout identity is invalid.')); } return yield* withExclusiveFileLock( fs, @@ -613,7 +617,7 @@ export const purgeCodeGraphIndex = Effect.fn('codeGraph.purgeIndex')(function* ( const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; if (!/^[0-9a-f]{64}$/.test(checkoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity is invalid.')); + return yield* Effect.fail(new CodeGraphMaintenanceError('Code graph checkout identity is invalid.')); } const waitTimeoutMilliseconds = options.waitTimeoutMilliseconds ?? 0; const lockOptions = {...CODE_GRAPH_PURGE_LOCK_OPTIONS, waitTimeoutMilliseconds}; @@ -642,7 +646,9 @@ export const purgeCodeGraphIndex = Effect.fn('codeGraph.purgeIndex')(function* ( yield* options.interlock?.beforeVerification?.() ?? Effect.void; const verified = yield* inspectCodeGraphIndexPurgeTarget(fs, path, threadnoteHome, checkoutId); if (!sameCodeGraphIndexPurgeTarget(initial, verified)) { - return yield* Effect.fail(new Error('Code graph checkout target changed before purge.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Code graph checkout target changed before purge.'), + ); } if (verified === undefined || options.dryRun) { return {existed: verified !== undefined, quarantine: undefined}; @@ -657,7 +663,9 @@ export const purgeCodeGraphIndex = Effect.fn('codeGraph.purgeIndex')(function* ( const moved = yield* inspectQuarantinedCodeGraphIndexPurgeTarget(fs, quarantine); if (!sameCodeGraphIndexPurgeTarget(verified, moved)) { yield* restoreQuarantinedCodeGraphIndexPurgeTarget(fs, quarantine, verified.path); - return yield* Effect.fail(new Error('Code graph checkout target changed before purge.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Code graph checkout target changed before purge.'), + ); } return {existed: true, quarantine}; }), @@ -819,12 +827,14 @@ function openCodeGraphIndexPurgeTarget( const openedInfo = yield* opened.stat; const openedIno = Option.getOrUndefined(openedInfo.ino); if (openedInfo.type !== 'Directory' || openedIno === undefined) { - return yield* Effect.fail(new Error('Refusing code graph purge without stable checkout identity metadata.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge without stable checkout identity metadata.'), + ); } const target = {dev: openedInfo.dev, ino: openedIno, path: planned.path} satisfies CodeGraphIndexPurgeTarget; const current = yield* inspectCodeGraphIndexPurgeTarget(fs, path, threadnoteHome, checkoutId); if (!sameCodeGraphIndexPurgeTarget(target, current)) { - return yield* Effect.fail(new Error('Code graph checkout target changed before purge.')); + return yield* Effect.fail(new CodeGraphMaintenanceError('Code graph checkout target changed before purge.')); } return target; }); @@ -840,23 +850,29 @@ function inspectCodeGraphIndexPurgeTarget( const repositories = codeGraphRepositoriesRoot(path, threadnoteHome); if (!(yield* fs.exists(repositories))) return undefined; if (yield* isSymbolicLink(fs, repositories)) { - return yield* Effect.fail(new Error('Refusing code graph purge through a symbolic-link repositories root.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge through a symbolic-link repositories root.'), + ); } const repositoriesInfo = yield* fs.stat(repositories).pipe(Effect.option); if (repositoriesInfo._tag === 'None' || repositoriesInfo.value.type !== 'Directory') { return yield* Effect.fail( - new Error('Refusing code graph purge because the repositories root is not a directory.'), + new CodeGraphMaintenanceError('Refusing code graph purge because the repositories root is not a directory.'), ); } const canonicalRepositories = yield* fs.realPath(repositories); const repositoryRoot = codeGraphRepositoryRoot(path, threadnoteHome, checkoutId); if (!(yield* fs.exists(repositoryRoot))) return undefined; if (yield* isSymbolicLink(fs, repositoryRoot)) { - return yield* Effect.fail(new Error('Refusing code graph purge through a symbolic-link checkout root.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge through a symbolic-link checkout root.'), + ); } const repositoryInfo = yield* fs.stat(repositoryRoot).pipe(Effect.option); if (repositoryInfo._tag === 'None' || repositoryInfo.value.type !== 'Directory') { - return yield* Effect.fail(new Error('Refusing code graph purge because the checkout root is not a directory.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge because the checkout root is not a directory.'), + ); } const canonicalRepository = yield* fs.realPath(repositoryRoot); if ( @@ -864,11 +880,15 @@ function inspectCodeGraphIndexPurgeTarget( path.basename(canonicalRepository) !== checkoutId || !isContained(path, canonicalRepositories, canonicalRepository) ) { - return yield* Effect.fail(new Error('Refusing code graph purge outside the repositories root.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge outside the repositories root.'), + ); } const ino = Option.getOrUndefined(repositoryInfo.value.ino); if (ino === undefined) { - return yield* Effect.fail(new Error('Refusing code graph purge without stable checkout identity metadata.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing code graph purge without stable checkout identity metadata.'), + ); } return {dev: repositoryInfo.value.dev, ino, path: canonicalRepository}; }); @@ -884,7 +904,7 @@ function inspectQuarantinedCodeGraphIndexPurgeTarget( if (Option.isNone(info) || info.value.type !== 'Directory') return undefined; const ino = Option.getOrUndefined(info.value.ino); return ino === undefined ? undefined : {dev: info.value.dev, ino, path: quarantine}; - }).pipe(Effect.catch(() => Effect.succeed(undefined))); + }); } function sameCodeGraphIndexPurgeTarget( @@ -916,7 +936,7 @@ function isSymbolicLink(fs: FileSystem.FileSystem, candidate: string): Effect.Ef function refuseUnsafeObsoleteInventory(inventory: ObsoleteCodeGraphStoreInventory): Effect.Effect { return inventory.unsafeEntryCount > 0 ? Effect.fail( - new Error( + new CodeGraphMaintenanceError( `Refusing obsolete code graph cleanup: ${inventory.unsafeEntryCount} obsolete-shaped entry/entries are symbolic links, non-files, or outside the checkout root.`, ), ) @@ -933,19 +953,27 @@ function verifyObsoletePurgeTarget( return Effect.gen(function* () { const parsed = obsoleteGraphFileName(file.fileName); if (!parsed || parsed.schemaVersion !== file.schemaVersion || parsed.kind !== file.kind) { - return yield* Effect.fail(new Error(`Refusing unexpected obsolete graph target ${file.fileName}.`)); + return yield* Effect.fail( + new CodeGraphMaintenanceError(`Refusing unexpected obsolete graph target ${file.fileName}.`), + ); } const repositoryRoot = codeGraphRepositoryRoot(path, threadnoteHome, checkoutId); if (yield* isSymbolicLink(fs, repositoryRoot)) { - return yield* Effect.fail(new Error('Refusing obsolete graph cleanup through a symbolic-link checkout root.')); + return yield* Effect.fail( + new CodeGraphMaintenanceError('Refusing obsolete graph cleanup through a symbolic-link checkout root.'), + ); } const candidate = path.join(repositoryRoot, file.fileName); if (yield* isSymbolicLink(fs, candidate)) { - return yield* Effect.fail(new Error(`Refusing symbolic-link obsolete graph target ${file.fileName}.`)); + return yield* Effect.fail( + new CodeGraphMaintenanceError(`Refusing symbolic-link obsolete graph target ${file.fileName}.`), + ); } const info = yield* fs.stat(candidate).pipe(Effect.option); if (info._tag === 'None' || info.value.type !== 'File') { - return yield* Effect.fail(new Error(`Obsolete graph target changed before cleanup: ${file.fileName}.`)); + return yield* Effect.fail( + new CodeGraphMaintenanceError(`Obsolete graph target changed before cleanup: ${file.fileName}.`), + ); } const canonicalRoot = yield* fs.realPath(repositoryRoot); const canonical = yield* fs.realPath(candidate); @@ -954,7 +982,9 @@ function verifyObsoletePurgeTarget( path.dirname(canonical) !== canonicalRoot || !isContained(path, canonicalRoot, canonical) ) { - return yield* Effect.fail(new Error(`Obsolete graph target escaped its checkout root: ${file.fileName}.`)); + return yield* Effect.fail( + new CodeGraphMaintenanceError(`Obsolete graph target escaped its checkout root: ${file.fileName}.`), + ); } }); } diff --git a/src/code_graph/maintenance_coordinator.ts b/src/code_graph/maintenance_coordinator.ts index f48db995..df4a8552 100644 --- a/src/code_graph/maintenance_coordinator.ts +++ b/src/code_graph/maintenance_coordinator.ts @@ -205,7 +205,7 @@ export class CodeGraphMaintenanceCoordinator extends Context.Service< Effect.gen(function* () { const inspected = yield* inspectCodeGraphViewDatabaseTarget(input.threadnoteHome, input.checkoutId); if (inspected.state !== 'ready' || inspected.databasePath !== input.databasePath) { - return yield* Effect.fail(new Error('Code graph cleanup database target changed.')); + return yield* Effect.fail(new CodeGraphStoreError('Code graph cleanup database target changed.')); } if (yield* codeGraphMaintenanceIntentActive(input.threadnoteHome)) { return yield* Effect.fail(new CodeGraphMaintenanceActiveError()); @@ -259,8 +259,8 @@ export class CodeGraphMaintenanceCoordinator extends Context.Service< Effect.provideService(Path.Path, path), Effect.provideService(SystemInfo, system), ), - monotonicMilliseconds: () => Effect.sync(() => performance.now()), - nowMilliseconds: () => Clock.currentTimeMillis, + monotonicMilliseconds: Effect.sync(() => performance.now()), + nowMilliseconds: Clock.currentTimeMillis, sleep: milliseconds => Effect.sleep(milliseconds), update: (input, entry, update) => store.updateRemovedViewCleanup(input.databasePath, entry, update, { @@ -329,7 +329,9 @@ export class CodeGraphMaintenanceCoordinator extends Context.Service< Effect.gen(function* () { const inspected = yield* inspectCodeGraphViewDatabaseTarget(input.threadnoteHome, input.checkoutId); if (inspected.state !== 'ready' || inspected.databasePath !== input.databasePath) { - return yield* Effect.fail(new Error('Code graph database target changed before index preparation.')); + return yield* Effect.fail( + new CodeGraphStoreError('Code graph database target changed before index preparation.'), + ); } if (yield* codeGraphMaintenanceIntentActive(input.threadnoteHome)) { return yield* Effect.fail(new CodeGraphMaintenanceActiveError()); diff --git a/src/code_graph/maintenance_gate.ts b/src/code_graph/maintenance_gate.ts index 420829d0..fd59d68a 100644 --- a/src/code_graph/maintenance_gate.ts +++ b/src/code_graph/maintenance_gate.ts @@ -34,6 +34,10 @@ interface MaintenanceIntentOwner { readonly token: string; } +class CodeGraphMaintenanceGateError extends Error { + readonly _tag = 'CodeGraphMaintenanceGateError' as const; +} + export const CODE_GRAPH_MAINTENANCE_PROGRESS_PHASES = [ 'acquiring-gates', 'waiting-builders', @@ -149,7 +153,9 @@ const withCodeGraphMaintenanceIntentOwner = Effect.fn('codeGraph.withMaintenance const intent = codeGraphMaintenanceIntentPath(path, threadnoteHome); const processStartIdentity = yield* system.processStartIdentity(system.processId); if (!processStartIdentity) { - return yield* Effect.fail(new Error('Could not identify the maintenance process instance.')); + return yield* Effect.fail( + new CodeGraphMaintenanceGateError('Could not identify the maintenance process instance.'), + ); } const owner = { processId: system.processId, @@ -325,10 +331,10 @@ function codeGraphWorktreeLockFiles( return Effect.gen(function* () { if (!(yield* fs.exists(root))) return []; if (Option.isSome(yield* fs.readLink(root).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph worktree lock root is a symbolic link.')); + return yield* Effect.fail(new CodeGraphMaintenanceGateError('Code graph worktree lock root is a symbolic link.')); } if ((yield* fs.stat(root)).type !== 'Directory') { - return yield* Effect.fail(new Error('Code graph worktree lock root is not a directory.')); + return yield* Effect.fail(new CodeGraphMaintenanceGateError('Code graph worktree lock root is not a directory.')); } return (yield* fs.readDirectory(root)) .filter(name => /^[0-9a-f]{64}\.lock$/.test(name)) @@ -388,7 +394,7 @@ const validateReportedMaintenance = Effect.fn('codeGraph.validateReportedMainten progress.total <= 0 || progress.completed > progress.total ) { - return yield* Effect.fail(new Error('Code graph maintenance progress is invalid.')); + return yield* Effect.fail(new CodeGraphMaintenanceGateError('Code graph maintenance progress is invalid.')); } }); @@ -419,10 +425,12 @@ const writeMaintenanceStatus = Effect.fn('codeGraph.writeMaintenanceStatus')(fun } satisfies StoredCodeGraphMaintenanceStatus; const content = `${JSON.stringify(status)}\n`; if (new TextEncoder().encode(content).byteLength > CODE_GRAPH_MAINTENANCE_STATUS_BYTES) { - return yield* Effect.fail(new Error('Code graph maintenance progress exceeded its bounded size.')); + return yield* Effect.fail( + new CodeGraphMaintenanceGateError('Code graph maintenance progress exceeded its bounded size.'), + ); } if (Option.isSome(yield* fs.readLink(statusPath).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph maintenance status is a symbolic link.')); + return yield* Effect.fail(new CodeGraphMaintenanceGateError('Code graph maintenance status is a symbolic link.')); } const temporary = path.join( path.dirname(statusPath), @@ -431,10 +439,14 @@ const writeMaintenanceStatus = Effect.fn('codeGraph.writeMaintenanceStatus')(fun yield* fs.writeFileString(temporary, content, {flag: 'wx', mode: 0o600}); yield* Effect.gen(function* () { if ((yield* fs.readFileString(intentPath)).trim() !== ownerToken) { - return yield* Effect.fail(new Error('Code graph maintenance owner changed before progress publication.')); + return yield* Effect.fail( + new CodeGraphMaintenanceGateError('Code graph maintenance owner changed before progress publication.'), + ); } if (Option.isSome(yield* fs.readLink(statusPath).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph maintenance status changed before publication.')); + return yield* Effect.fail( + new CodeGraphMaintenanceGateError('Code graph maintenance status changed before publication.'), + ); } yield* fs.rename(temporary, statusPath); }).pipe(Effect.onError(() => fs.remove(temporary, {force: true}).pipe(Effect.catch(() => Effect.void)))); diff --git a/src/code_graph/manager_catalog_revision.ts b/src/code_graph/manager_catalog_revision.ts index 04d7f6a7..bf55a24c 100644 --- a/src/code_graph/manager_catalog_revision.ts +++ b/src/code_graph/manager_catalog_revision.ts @@ -75,7 +75,7 @@ export const observeManagerGraphCatalogStatus = Effect.fn('codeGraph.observeMana checkoutId, repositoryId: view.repositoryId, worktreeId: view.worktreeId, - }).pipe(Effect.catch(() => Effect.succeed({available: false, state: 'invalid'} as const))), + }), {concurrency: 2}, ); const anchor = associations.find(association => association.state === 'verified' && 'path' in association); diff --git a/src/code_graph/manager_status.ts b/src/code_graph/manager_status.ts index cd3fedae..dc9275fa 100644 --- a/src/code_graph/manager_status.ts +++ b/src/code_graph/manager_status.ts @@ -1,4 +1,5 @@ import {Effect, Option} from 'effect'; +import type {CodeGraphAutomaticCompactionStatus} from './automatic_compaction.js'; import { readAllCodeGraphBuildStatuses, selectCodeGraphBuildStatuses, @@ -8,19 +9,89 @@ import {runCodeGraphLifecycleOpportunity} from './lifecycle_opportunity.js'; import {observeManagerGraphCatalogStatus} from './manager_catalog_revision.js'; import {CodeGraphMaintenanceCoordinator} from './maintenance_coordinator.js'; import {observeCodeGraphMaintenanceStatus, type CodeGraphMaintenanceStatus} from './maintenance_gate.js'; +import {compareCodeUnits} from './ordering.js'; +import {codeGraphCompactionRequiredFreeBytes, inspectCodeGraphStorage, type CodeGraphStorage} from './storage.js'; + +export const MANAGER_GRAPH_STORAGE_STATUS_LIMIT = 8; + +export type ManagerGraphPageStorageSummary = + | { + readonly allocatedBytes: number; + readonly automaticCompaction: 'eligible' | 'not-needed' | 'space-unknown' | 'waiting-for-space'; + readonly compactionOpportunityBytes?: number; + readonly inUseBytes: number; + readonly reclaimableRatio: number; + readonly requiredFreeBytes?: number; + readonly reusableBytes: number; + readonly state: 'available'; + } + | {readonly reason: 'active-build'; readonly state: 'deferred'} + | {readonly reason: 'database-busy-or-unreadable'; readonly state: 'unavailable'}; + +export type ManagerGraphStorageSummary = + | {readonly state: 'missing' | 'unavailable'} + | { + readonly databaseBytes: number; + readonly pageStorage: ManagerGraphPageStorageSummary; + readonly physicalBytes: number; + readonly sidecarBytes: number; + readonly state: 'available'; + }; + +/** Privacy-safe storage totals for Manager; the database path never crosses the API boundary. */ +export function managerGraphStorageSummary(storage: CodeGraphStorage): ManagerGraphStorageSummary { + if (storage.state === 'missing') return {state: 'missing'}; + const pageStorage: ManagerGraphPageStorageSummary = (() => { + if (storage.pageStorage.state !== 'available') return storage.pageStorage; + const allocatedBytes = storage.pageStorage.pageCount * storage.pageStorage.pageSize; + const requiredFreeBytes = codeGraphCompactionRequiredFreeBytes(storage); + const automaticCompaction = + storage.pageStorage.threshold.reason !== 'freelist' + ? 'not-needed' + : storage.availableBytes === undefined + ? 'space-unknown' + : storage.availableBytes >= requiredFreeBytes + ? 'eligible' + : 'waiting-for-space'; + return { + allocatedBytes, + automaticCompaction, + ...(storage.pageStorage.compactionOpportunityBytes === undefined + ? {} + : {compactionOpportunityBytes: storage.pageStorage.compactionOpportunityBytes}), + inUseBytes: Math.max(0, allocatedBytes - storage.pageStorage.reclaimableBytes), + reclaimableRatio: storage.pageStorage.reclaimableRatio, + ...(automaticCompaction === 'waiting-for-space' ? {requiredFreeBytes} : {}), + reusableBytes: storage.pageStorage.reclaimableBytes, + state: 'available', + }; + })(); + return { + databaseBytes: storage.databaseBytes, + pageStorage, + physicalBytes: storage.filesystemBytes, + sidecarBytes: Math.max(0, storage.filesystemBytes - storage.databaseBytes), + state: 'available', + }; +} export interface ManagerGraphBuildCatalog { + readonly automaticCompaction?: CodeGraphAutomaticCompactionStatus; readonly builds: readonly ObservedCodeGraphBuildStatus[]; readonly catalogRevision?: string; readonly lifecyclePending: boolean; readonly maintenance?: CodeGraphMaintenanceStatus; readonly queuedWorktreeIds: readonly string[]; + readonly storage: Readonly>; readonly waiterCount: number; readonly waiters: readonly ObservedCodeGraphBuildStatus[]; } /** Bounded live status plus one non-tailing missing-view reconciliation opportunity. */ -export const managerGraphBuildCatalog = Effect.fn('codeGraph.managerBuildCatalog')(function* (threadnoteHome: string) { +export const managerGraphBuildCatalog = Effect.fn('codeGraph.managerBuildCatalog')(function* ( + threadnoteHome: string, + automaticCompaction?: CodeGraphAutomaticCompactionStatus, +) { const selection = selectCodeGraphBuildStatuses(yield* readAllCodeGraphBuildStatuses(threadnoteHome)); const maintenance = yield* observeCodeGraphMaintenanceStatus(threadnoteHome).pipe( Effect.catch(() => Effect.succeed(undefined)), @@ -51,13 +122,54 @@ export const managerGraphBuildCatalog = Effect.fn('codeGraph.managerBuildCatalog } } const catalogRevision = statusObservation?.catalogRevision; + const checkoutIds = managerGraphStorageStatusCheckoutIds([...selection.builds, ...selection.waiters]); + const storage = Object.fromEntries( + yield* Effect.forEach( + checkoutIds, + checkoutId => + inspectCodeGraphStorage(threadnoteHome, checkoutId).pipe( + Effect.map(observation => [checkoutId, managerGraphStorageSummary(observation)] as const), + Effect.catch(() => Effect.succeed([checkoutId, {state: 'unavailable'}] as const)), + ), + {concurrency: 2}, + ), + ); return { + ...(automaticCompaction === undefined ? {} : {automaticCompaction}), builds: selection.builds, ...(catalogRevision === undefined ? {} : {catalogRevision}), lifecyclePending: statusObservation?.lifecyclePending === true, ...(maintenance === undefined ? {} : {maintenance}), queuedWorktreeIds: [...new Set(selection.waiters.map(status => status.identity.worktreeId))], + storage, waiterCount: selection.waiters.length, waiters: selection.waiters, } satisfies ManagerGraphBuildCatalog; }); + +/** Active builds win the bounded storage-inspection budget; recent receipts follow deterministically. */ +export function managerGraphStorageStatusCheckoutIds( + statuses: readonly { + readonly identity: Pick; + readonly state: ObservedCodeGraphBuildStatus['state']; + readonly timestamps: Pick; + }[], +): readonly string[] { + const ordered = [...statuses].sort((left, right) => { + const leftActive = left.state === 'queued' || left.state === 'running'; + const rightActive = right.state === 'queued' || right.state === 'running'; + if (leftActive !== rightActive) return leftActive ? -1 : 1; + const byRecency = compareCodeUnits(right.timestamps.lastProgressAt, left.timestamps.lastProgressAt); + return byRecency || compareCodeUnits(left.identity.checkoutId, right.identity.checkoutId); + }); + const checkoutIds: string[] = []; + const seen = new Set(); + for (const status of ordered) { + const {checkoutId} = status.identity; + if (!/^[0-9a-f]{64}$/u.test(checkoutId) || seen.has(checkoutId)) continue; + seen.add(checkoutId); + checkoutIds.push(checkoutId); + if (checkoutIds.length >= MANAGER_GRAPH_STORAGE_STATUS_LIMIT) break; + } + return checkoutIds; +} diff --git a/src/code_graph/parser_worker.ts b/src/code_graph/parser_worker.ts index b7a4b9a9..fc47e0ed 100644 --- a/src/code_graph/parser_worker.ts +++ b/src/code_graph/parser_worker.ts @@ -1,6 +1,6 @@ import {Context, Crypto, Effect, FileSystem, Layer, Option, Path, Queue, Stdio, Stream} from 'effect'; import {sha256HexSync} from '../crypto/sha256.js'; -import {fromPromiseError, fromPromiseInterruptible} from '../effect/errors.js'; +import {fromPromise, fromPromiseInterruptible} from '../effect/errors.js'; import {isFileLockTimeout, withExclusiveFileLock} from '../effect/file_lock.js'; import {SystemInfo, type SystemInfoShape} from '../effect/system.js'; import { @@ -158,7 +158,7 @@ export interface CodeGraphParserPoolShape { file: CodeGraphInventoryFile, threadnoteHome: string, ) => Effect.Effect; - readonly trimIdle: () => Effect.Effect; + readonly trimIdle: Effect.Effect; } export interface ParserWorkerCapacityInput { @@ -191,7 +191,7 @@ export function codeGraphParserPoolLayer( const explicitCapacity = explicitParserWorkerCapacity(environment, options.capacity); const capacity = explicitCapacity ?? - (yield* system.hardwareInfo().pipe( + (yield* system.hardwareInfo.pipe( Effect.map(hardware => parserWorkerCapacity({ effectiveMemoryBytes: hardware.effectiveMemoryBytes, @@ -275,22 +275,21 @@ export function codeGraphParserPoolLayer( slot => Queue.offer(available, slot), ); }, - trimIdle: () => - Effect.acquireUseRelease( - Queue.clear(available), - idleSlots => - Effect.forEach(idleSlots, slot => fromPromiseError(() => slot.trimIdle()), { - concurrency: 'unbounded', - discard: true, - }).pipe(Effect.catch(() => Effect.void)), - idleSlots => Queue.offerAll(available, idleSlots), - ).pipe(Effect.asVoid), + trimIdle: Effect.acquireUseRelease( + Queue.clear(available), + idleSlots => + Effect.forEach(idleSlots, slot => fromPromise('trim idle parser worker', () => slot.trimIdle()), { + concurrency: 'unbounded', + discard: true, + }).pipe(Effect.catch(() => Effect.void)), + idleSlots => Queue.offerAll(available, idleSlots), + ).pipe(Effect.asVoid), }), slots, }; }), ({slots}) => - Effect.forEach(slots, slot => fromPromiseError(() => slot.close()), { + Effect.forEach(slots, slot => fromPromise('close parser worker', () => slot.close()), { concurrency: 'unbounded', discard: true, }).pipe(Effect.catch(() => Effect.void)), diff --git a/src/code_graph/removed_view_build_cleanup.ts b/src/code_graph/removed_view_build_cleanup.ts index 5540f8cd..449f0f28 100644 --- a/src/code_graph/removed_view_build_cleanup.ts +++ b/src/code_graph/removed_view_build_cleanup.ts @@ -65,7 +65,9 @@ type BuildStatusCursor = readonly mode: 'verify'; }; -class InvalidBuildSidecarError extends Error {} +class InvalidBuildSidecarError extends Error { + readonly _tag = 'InvalidBuildSidecarError' as const; +} /** * Remove at most one exact terminal status for the tombstoned snapshot. diff --git a/src/code_graph/removed_view_cleanup.ts b/src/code_graph/removed_view_cleanup.ts index d6ea94b2..abc994c8 100644 --- a/src/code_graph/removed_view_cleanup.ts +++ b/src/code_graph/removed_view_cleanup.ts @@ -78,8 +78,8 @@ export interface CodeGraphRemovedViewCleanupWorkerDependencies { use: (commit: Effect.Effect) => Effect.Effect, ) => Effect.Effect; /** Monotonic elapsed-time source; never use wall clock for the burst deadline. */ - readonly monotonicMilliseconds: () => Effect.Effect; - readonly nowMilliseconds: () => Effect.Effect; + readonly monotonicMilliseconds: Effect.Effect; + readonly nowMilliseconds: Effect.Effect; readonly sleep: (milliseconds: number) => Effect.Effect; readonly update: ( input: CodeGraphRemovedViewCleanupWorkerInput, @@ -132,7 +132,7 @@ export const makeCodeGraphRemovedViewCleanupWorker = Effect.fn('codeGraph.makeRe ): Effect.Effect => Effect.gen(function* () { const result: MutableWorkerResult = {advanced: 0, claimed: 0, deferred: 0, progressed: 0, stale: 0}; - const startedAt = yield* dependencies.monotonicMilliseconds(); + const startedAt = yield* dependencies.monotonicMilliseconds; const preparation: CodeGraphRemovedViewCleanupVectorPreparation = { deadlineMonotonicMilliseconds: startedAt + maximumDurationMilliseconds, reservationMode: 'nonblocking-one-attempt', @@ -142,10 +142,10 @@ export const makeCodeGraphRemovedViewCleanupWorker = Effect.fn('codeGraph.makeRe while (result.claimed < maximumUnits) { if (result.claimed > 0) { - const observedAt = yield* dependencies.monotonicMilliseconds(); + const observedAt = yield* dependencies.monotonicMilliseconds; if (observedAt - startedAt >= maximumDurationMilliseconds) break; } - const claimAt = yield* dependencies.nowMilliseconds(); + const claimAt = yield* dependencies.nowMilliseconds; const claimed = yield* dependencies.claim(input, claimAt, 1).pipe( Effect.match({ onFailure: () => undefined, @@ -166,7 +166,7 @@ export const makeCodeGraphRemovedViewCleanupWorker = Effect.fn('codeGraph.makeRe const outcome = yield* runClaimedUnit(dependencies, input, entry, preparation); result[outcome] += 1; if (result.claimed < maximumUnits) { - const beforePause = yield* dependencies.monotonicMilliseconds(); + const beforePause = yield* dependencies.monotonicMilliseconds; const remainingMilliseconds = maximumDurationMilliseconds - (beforePause - startedAt); if (remainingMilliseconds <= 0) break; yield* dependencies.sleep( @@ -183,7 +183,7 @@ export const makeCodeGraphRemovedViewCleanupWorker = Effect.fn('codeGraph.makeRe }; } return {...result, remaining, state: 'worked' as const}; - }).pipe(Effect.catch(() => Effect.succeed(unavailableResult()))); + }); return { burst: input => @@ -236,7 +236,7 @@ const runAuthorizedCleanupUnit = Effect.fn('codeGraph.runAuthorizedRemovedViewCl const page = yield* cleanup(authorization.entry).pipe(Effect.catch(() => Effect.succeed(ioFailurePage()))); const normalized = normalizePageResult(authorization.entry, page); - const now = yield* dependencies.nowMilliseconds(); + const now = yield* dependencies.nowMilliseconds; const update = updateForPageResult(authorization.entry, normalized, now); if (update === undefined) return 'deferred' as const; const stored = yield* dependencies.update(input, authorization.entry, update); @@ -374,15 +374,3 @@ function invalidSidecarPage(): CodeGraphRemovedViewCleanupPageResult { function ioFailurePage(): CodeGraphRemovedViewCleanupPageResult { return {blockedCode: 'io-error', retryAfterMilliseconds: 1_000, state: 'deferred'}; } - -function unavailableResult(): CodeGraphRemovedViewCleanupWorkerResult { - return { - advanced: 0, - claimed: 0, - deferred: 0, - progressed: 0, - remaining: true, - stale: 0, - state: 'deferred', - }; -} diff --git a/src/code_graph/repository.ts b/src/code_graph/repository.ts index a5f2ad35..062e9638 100644 --- a/src/code_graph/repository.ts +++ b/src/code_graph/repository.ts @@ -35,7 +35,7 @@ export const resolveRepositoryIdentityDetail = Effect.fn('codeGraph.resolveRepos Effect.mapError(cause => new CodeGraphRepositoryError(`Not a Git repository: ${cause.message}`)), ); const repoRoot = yield* fs.realPath(rootResult.stdout.trim()); - const [directoryResult, formatResult, commitResult, ignoreCaseResult, remoteResult] = yield* Effect.all( + const [directoryResult, formatResult, commitResult, ignoreCaseResult, remoteResult, branch] = yield* Effect.all( [ runBinaryCommandEffect( 'git', @@ -46,8 +46,9 @@ export const resolveRepositoryIdentityDetail = Effect.fn('codeGraph.resolveRepos runGit(repoRoot, ['rev-parse', 'HEAD'], true), runGit(repoRoot, ['config', '--bool', 'core.ignorecase'], true), runGit(repoRoot, ['remote', 'get-url', 'origin'], true), + observeRepositoryBranch(repoRoot), ], - {concurrency: 5}, + {concurrency: 6}, ); const directories = parseGitDirectoryOutput(directoryResult.stdout); if (directories === undefined) { @@ -66,6 +67,7 @@ export const resolveRepositoryIdentityDetail = Effect.fn('codeGraph.resolveRepos const headCommit = commitResult.exitCode === 0 ? commitResult.stdout.trim() : zeroObjectId(objectFormat); const displayName = repositoryDisplayName(remoteIdentity, repoRoot); const identity = { + ...(branch.state === 'current' ? {branch: branch.branch} : {}), caseMode: ignoreCaseResult.exitCode === 0 && ignoreCaseResult.stdout.trim().toLowerCase() === 'true' ? 'insensitive' @@ -83,13 +85,45 @@ export const resolveRepositoryIdentityDetail = Effect.fn('codeGraph.resolveRepos return {gitDirectory: directories.gitDirectory, identity}; }); +export function normalizeRepositoryBranchName(value: string): string | undefined { + const branch = value.trim(); + return branch.length > 0 && + new TextEncoder().encode(branch).byteLength <= 1_024 && + !hasControlCharacter(branch) && + !/\p{Bidi_Control}/u.test(branch) + ? branch + : undefined; +} + +export const observeRepositoryBranch = Effect.fn('codeGraph.observeRepositoryBranch')(function* (cwd: string) { + const result = yield* runCommandEffect('git', ['-C', cwd, 'symbolic-ref', '--quiet', '--short', 'HEAD'], { + allowFailure: true, + maxOutputBytes: 2_048, + timeoutMs: 5_000, + }).pipe(Effect.option); + if (result._tag === 'None' || (result.value.exitCode !== 0 && result.value.exitCode !== 1)) { + return {state: 'missing' as const}; + } + if (result.value.exitCode === 1) return {state: 'detached' as const}; + const branch = normalizeRepositoryBranchName(result.value.stdout); + return branch === undefined ? {state: 'missing' as const} : {branch, state: 'current' as const}; +}); + +function hasControlCharacter(value: string): boolean { + for (const character of value) { + const codePoint = character.codePointAt(0); + if (codePoint !== undefined && (codePoint <= 31 || codePoint === 127)) return true; + } + return false; +} + export const resolveRepositoryIdentity = Effect.fn('codeGraph.resolveRepositoryIdentity')(function* (cwd: string) { return (yield* resolveRepositoryIdentityDetail(cwd)).identity; }); /** * Revalidate a previously published repository identity without repeating the - * six-command discovery path. The expected repository ID remains independently + * full discovery path. The expected repository ID remains independently * recomputed from the current remote or local checkout before it is trusted. */ export const resolveRepositoryIdentityForExpectation = Effect.fn('codeGraph.resolveRepositoryIdentityForExpectation')( @@ -97,7 +131,7 @@ export const resolveRepositoryIdentityForExpectation = Effect.fn('codeGraph.reso const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const system = yield* SystemInfo; - const [metadataResult, ignoreCaseResult, remoteResult] = yield* Effect.all( + const [metadataResult, ignoreCaseResult, remoteResult, branch] = yield* Effect.all( [ runGit(cwd, [ 'rev-parse', @@ -109,8 +143,9 @@ export const resolveRepositoryIdentityForExpectation = Effect.fn('codeGraph.reso ]), runGit(cwd, ['config', '--bool', 'core.ignorecase'], true), runGit(cwd, ['remote', 'get-url', 'origin'], true), + observeRepositoryBranch(cwd), ], - {concurrency: 3}, + {concurrency: 4}, ).pipe(Effect.mapError(() => new CodeGraphRepositoryError('Repository identity could not be revalidated.'))); const metadata = metadataResult.stdout.replace(/\r?\n$/u, '').split(/\r?\n/u); if (metadata.length !== 4) { @@ -132,6 +167,7 @@ export const resolveRepositoryIdentityForExpectation = Effect.fn('codeGraph.reso remoteResult.exitCode === 0 ? normalizeCredentialFreeRemote(remoteResult.stdout.trim()) : undefined; const repositorySource = remoteIdentity ?? `local:${normalizeLocalIdentity(gitCommonDirectory, system.platform)}`; const identity = { + ...(branch.state === 'current' ? {branch: branch.branch} : {}), caseMode: ignoreCaseResult.exitCode === 0 && ignoreCaseResult.stdout.trim().toLowerCase() === 'true' ? 'insensitive' diff --git a/src/code_graph/snapshot_purge.ts b/src/code_graph/snapshot_purge.ts index f9ba04ca..04c606b1 100644 --- a/src/code_graph/snapshot_purge.ts +++ b/src/code_graph/snapshot_purge.ts @@ -23,6 +23,10 @@ import { } from './vector_maintenance.js'; import {inspectCodeGraphViewDatabaseTarget} from './view_removal.js'; +class CodeGraphSnapshotPurgeError extends Error { + readonly _tag = 'CodeGraphSnapshotPurgeError' as const; +} + const HASH_ID = /^[0-9a-f]{64}$/u; const SNAPSHOT_ID = /^cgsn_[0-9a-f]{40}(?:-direct|-full-[0-9a-f]{16})?$/u; const APPROVAL_DIGEST = /^sha256:[0-9a-f]{64}$/u; @@ -177,7 +181,11 @@ export const purgeCodeGraphSnapshot = Effect.fn('codeGraph.purgeSnapshotAction') Effect.flatMap(currentTarget => currentTarget.state === 'ready' && currentTarget.databasePath === inspected.databasePath ? Effect.void - : Effect.fail(new Error('Code graph database target changed before snapshot purge.')), + : Effect.fail( + new CodeGraphSnapshotPurgeError( + 'Code graph database target changed before snapshot purge.', + ), + ), ), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -267,11 +275,16 @@ export function renderCodeGraphSnapshotPurgeResult(result: CodeGraphSnapshotPurg } export function codeGraphSnapshotPurgeTargetFailure(result: CodeGraphSnapshotPurgeActionResult): Error | undefined { - if (result.state === 'not-found') return new Error('The selected code graph snapshot does not exist.'); - if (result.state === 'approval-required') return new Error('A fresh snapshot purge approval digest is required.'); - if (result.state === 'state-changed') return new Error('The selected snapshot changed; preview it again.'); + if (result.state === 'not-found') + return new CodeGraphSnapshotPurgeError('The selected code graph snapshot does not exist.'); + if (result.state === 'approval-required') + return new CodeGraphSnapshotPurgeError('A fresh snapshot purge approval digest is required.'); + if (result.state === 'state-changed') + return new CodeGraphSnapshotPurgeError('The selected snapshot changed; preview it again.'); if (result.state === 'blocked') { - return new Error(`The selected snapshot is protected: ${result.blockers.map(blocker => blocker.code).join(', ')}.`); + return new CodeGraphSnapshotPurgeError( + `The selected snapshot is protected: ${result.blockers.map(blocker => blocker.code).join(', ')}.`, + ); } return undefined; } @@ -419,9 +432,11 @@ const validateSnapshotPurgeTarget = Effect.fn('codeGraph.validateSnapshotPurgeTa target: CodeGraphSnapshotPurgeTarget, ) { if (!HASH_ID.test(target.checkoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity must be 64 lowercase hexadecimal characters.')); + return yield* Effect.fail( + new CodeGraphSnapshotPurgeError('Code graph checkout identity must be 64 lowercase hexadecimal characters.'), + ); } if (!SNAPSHOT_ID.test(target.snapshotId)) { - return yield* Effect.fail(new Error('Code graph snapshot identity is invalid.')); + return yield* Effect.fail(new CodeGraphSnapshotPurgeError('Code graph snapshot identity is invalid.')); } }); diff --git a/src/code_graph/storage.ts b/src/code_graph/storage.ts index 95b2c70d..57b0bbd8 100644 --- a/src/code_graph/storage.ts +++ b/src/code_graph/storage.ts @@ -2,6 +2,7 @@ import {Database} from 'bun:sqlite'; import {Effect, FileSystem, Option, Path} from 'effect'; import {isFileLockTimeout, withExclusiveFileLock} from '../effect/file_lock.js'; import {SystemInfo, type SystemInfoShape} from '../effect/system.js'; +import {recordCodeGraphAutomaticCompactionAttempt} from './automatic_compaction_receipt.js'; import {codeGraphMaintenanceLockPath, codeGraphRepositoryLockPath, codeGraphRepositoryRoot} from './layout.js'; import { awaitCodeGraphWorktreeBuilds, @@ -17,6 +18,10 @@ import { type CodeGraphStorageSemanticAttribution, } from './storage_attribution.js'; +class CodeGraphStorageOperationError extends Error { + readonly _tag = 'CodeGraphStorageOperationError' as const; +} + export const CODE_GRAPH_COMPACTION_MIN_RECLAIMABLE_BYTES = 512 * 1024 * 1024; export const CODE_GRAPH_COMPACTION_MIN_RECLAIMABLE_RATIO = 0.2; export const CODE_GRAPH_COMPACTION_MIN_SAFETY_MARGIN_BYTES = 512 * 1024 * 1024; @@ -89,10 +94,11 @@ export function codeGraphStorageUnattributedBytes( ['attributed', attributedBytes], ['freelist', freelistBytes], ] as const) { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid SQLite ${label} storage bytes.`); + if (!Number.isSafeInteger(value) || value < 0) + throw new CodeGraphStorageOperationError(`Invalid SQLite ${label} storage bytes.`); } if (attributedBytes > allocatedBytes || freelistBytes > allocatedBytes - attributedBytes) { - throw new Error('SQLite storage attribution exceeds allocated page bytes.'); + throw new CodeGraphStorageOperationError('SQLite storage attribution exceeds allocated page bytes.'); } return allocatedBytes - attributedBytes - freelistBytes; } @@ -115,7 +121,8 @@ export function codeGraphCompactionRecommendation(input: { ['fragmented', fragmentedBytes], ['reclaimable', reclaimableBytes], ] as const) { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid SQLite ${label} storage bytes.`); + if (!Number.isSafeInteger(value) || value < 0) + throw new CodeGraphStorageOperationError(`Invalid SQLite ${label} storage bytes.`); } const compactionOpportunityBytes = reclaimableBytes + fragmentedBytes; if ( @@ -123,7 +130,7 @@ export function codeGraphCompactionRecommendation(input: { reclaimableBytes > allocatedBytes || compactionOpportunityBytes > allocatedBytes ) { - throw new Error('SQLite compaction opportunity exceeds allocated page bytes.'); + throw new CodeGraphStorageOperationError('SQLite compaction opportunity exceeds allocated page bytes.'); } const reclaimableRatio = allocatedBytes === 0 ? 0 : reclaimableBytes / allocatedBytes; const compactionOpportunityRatio = allocatedBytes === 0 ? 0 : compactionOpportunityBytes / allocatedBytes; @@ -358,7 +365,9 @@ export const compactCodeGraphStorage = Effect.fn('codeGraph.compactStorage')(fun } satisfies CodeGraphCompactionSummary; } if (before.pageStorage.state !== 'available') { - return yield* Effect.fail(new Error('Code graph page storage could not be inspected under its lock.')); + return yield* Effect.fail( + new CodeGraphStorageOperationError('Code graph page storage could not be inspected under its lock.'), + ); } if (!options.force && !before.pageStorage.threshold.recommended) { return { @@ -388,13 +397,17 @@ export const compactCodeGraphStorage = Effect.fn('codeGraph.compactStorage')(fun yield* vacuumDatabase(databasePath); const afterReceipt = yield* readCompactionReceipt(databasePath); if (!sameCompactionReceipt(receipt, afterReceipt)) { - return yield* Effect.fail(new Error('Code graph compaction changed the active snapshot receipt.')); + return yield* Effect.fail( + new CodeGraphStorageOperationError('Code graph compaction changed the active snapshot receipt.'), + ); } const after = yield* inspectCodeGraphStorage(threadnoteHome, checkoutId, {openWhileLocked: true}); if (after.state === 'missing') { - return yield* Effect.fail(new Error('Code graph database disappeared during compaction.')); + return yield* Effect.fail( + new CodeGraphStorageOperationError('Code graph database disappeared during compaction.'), + ); } - return { + const summary = { action: 'compacted', after, before, @@ -403,6 +416,12 @@ export const compactCodeGraphStorage = Effect.fn('codeGraph.compactStorage')(fun dryRun: false, reclaimedBytes: Math.max(0, before.databaseBytes - after.databaseBytes), } satisfies CodeGraphCompactionSummary; + yield* recordCodeGraphAutomaticCompactionAttempt( + threadnoteHome, + {checkoutId, opportunityBytes: codeGraphCompactionSummaryOpportunityBytes(summary)}, + {action: summary.action, reclaimedBytes: summary.reclaimedBytes}, + ); + return summary; }), 0, ); @@ -420,20 +439,45 @@ export const compactCodeGraphStorage = Effect.fn('codeGraph.compactStorage')(fun isFileLockTimeout(cause) ? Effect.succeed(deferred('active-maintenance')) : Effect.fail(cause), ), ); - return yield* maintain; + if (options.dryRun) return yield* maintain; + const candidate = (opportunityBytes: number) => ({checkoutId, opportunityBytes}); + return yield* maintain.pipe( + Effect.tap(summary => + summary.action === 'compacted' + ? Effect.void + : recordCodeGraphAutomaticCompactionAttempt( + threadnoteHome, + candidate(codeGraphCompactionSummaryOpportunityBytes(summary)), + {action: summary.action, reclaimedBytes: summary.reclaimedBytes}, + ), + ), + Effect.tapError(() => recordCodeGraphAutomaticCompactionAttempt(threadnoteHome, candidate(0), undefined)), + ); }); +function codeGraphCompactionSummaryOpportunityBytes(summary: CodeGraphCompactionSummary): number { + if (!('before' in summary) || summary.before === undefined || summary.before.pageStorage.state !== 'available') { + return 0; + } + return summary.before.pageStorage.compactionOpportunityBytes ?? summary.before.pageStorage.reclaimableBytes; +} + export function codeGraphCompactionRequiredFreeBytes( storage: Pick, ): number { - const sourceBytes = storage.databaseBytes + storage.walBytes; + // SQLite may need one database-sized temporary copy and another database-sized + // rollback journal while VACUUM copies the compacted image back. Existing WAL + // bytes and a separate safety margin remain additional conservative headroom. + const vacuumWorkingBytes = storage.databaseBytes * 2 + storage.walBytes; const safetyMargin = Math.max( CODE_GRAPH_COMPACTION_MIN_SAFETY_MARGIN_BYTES, - Math.ceil(sourceBytes * CODE_GRAPH_COMPACTION_SAFETY_MARGIN_RATIO), + Math.ceil(vacuumWorkingBytes * CODE_GRAPH_COMPACTION_SAFETY_MARGIN_RATIO), ); - const required = sourceBytes + safetyMargin; + const required = vacuumWorkingBytes + safetyMargin; if (!Number.isSafeInteger(required) || required < 0) { - throw new Error('Code graph compaction storage requirement exceeds the supported byte range.'); + throw new CodeGraphStorageOperationError( + 'Code graph compaction storage requirement exceeds the supported byte range.', + ); } return required; } @@ -449,7 +493,7 @@ const verifyCompactionDiskHeadroom = Effect.fn('codeGraph.verifyCompactionDiskHe .pipe( Effect.mapError( cause => - new Error( + new CodeGraphStorageOperationError( `Could not inspect free disk space before code graph compaction. ` + `Verify at least ${requiredBytes.toLocaleString()} bytes are free and retry; the database was not modified.`, {cause}, @@ -458,7 +502,7 @@ const verifyCompactionDiskHeadroom = Effect.fn('codeGraph.verifyCompactionDiskHe ); if (availableBytes === undefined) { return yield* Effect.fail( - new Error( + new CodeGraphStorageOperationError( `Could not determine free disk space before code graph compaction. ` + `Verify at least ${requiredBytes.toLocaleString()} bytes are free and retry; the database was not modified.`, ), @@ -466,7 +510,7 @@ const verifyCompactionDiskHeadroom = Effect.fn('codeGraph.verifyCompactionDiskHe } if (availableBytes < requiredBytes) { return yield* Effect.fail( - new Error( + new CodeGraphStorageOperationError( `Code graph compaction needs ${requiredBytes.toLocaleString()} bytes free, but only ` + `${availableBytes.toLocaleString()} bytes are available. Free disk space and retry; the database was not modified.`, ), @@ -480,16 +524,20 @@ function regularFileBytes( ): Effect.Effect<{readonly bytes: number; readonly state: 'available'} | {readonly state: 'missing'}, Error | unknown> { return Effect.gen(function* () { if (Option.isSome(yield* fs.readLink(candidate).pipe(Effect.option))) { - return yield* Effect.fail(new Error(`Refusing symbolic-link code graph storage path: ${candidate}`)); + return yield* Effect.fail( + new CodeGraphStorageOperationError(`Refusing symbolic-link code graph storage path: ${candidate}`), + ); } const info = yield* fs.stat(candidate).pipe(Effect.option); if (Option.isNone(info)) return {state: 'missing'} as const; if (info.value.type !== 'File') { - return yield* Effect.fail(new Error(`Code graph storage path is not a regular file: ${candidate}`)); + return yield* Effect.fail( + new CodeGraphStorageOperationError(`Code graph storage path is not a regular file: ${candidate}`), + ); } const bytes = Number(info.value.size); if (!Number.isSafeInteger(bytes) || bytes < 0) { - return yield* Effect.fail(new Error(`Code graph storage size is invalid: ${candidate}`)); + return yield* Effect.fail(new CodeGraphStorageOperationError(`Code graph storage size is invalid: ${candidate}`)); } return {bytes, state: 'available'} as const; }); @@ -547,7 +595,8 @@ function readPageStorage( database.close(false); } }, - catch: cause => new Error(`Could not inspect code graph page storage: ${errorText(cause)}`), + catch: cause => + new CodeGraphStorageOperationError(`Could not inspect code graph page storage: ${errorText(cause)}`), }); } @@ -645,13 +694,13 @@ function readCompactionReceipt(databasePath: string): Effect.Effect new Error(`Could not verify code graph compaction receipt: ${errorText(cause)}`), + catch: cause => + new CodeGraphStorageOperationError(`Could not verify code graph compaction receipt: ${errorText(cause)}`), }); } @@ -689,17 +739,18 @@ function vacuumDatabase(databasePath: string): Effect.Effect { readonly busy?: number; } | null; if (Number(before?.busy ?? 0) !== 0) - throw new Error('active SQLite readers prevented the preflight checkpoint'); + throw new CodeGraphStorageOperationError('active SQLite readers prevented the preflight checkpoint'); database.exec('VACUUM'); const after = database.query('PRAGMA wal_checkpoint(TRUNCATE)').get() as { readonly busy?: number; } | null; - if (Number(after?.busy ?? 0) !== 0) throw new Error('active SQLite readers prevented the final checkpoint'); + if (Number(after?.busy ?? 0) !== 0) + throw new CodeGraphStorageOperationError('active SQLite readers prevented the final checkpoint'); } finally { database.close(false); } }, - catch: cause => new Error(`Code graph compaction failed safely: ${errorText(cause)}`), + catch: cause => new CodeGraphStorageOperationError(`Code graph compaction failed safely: ${errorText(cause)}`), }); } @@ -710,13 +761,13 @@ function pragmaNumber(database: Database, pragma: 'freelist_count' | 'page_count function safeCount(value: bigint | number, label: string): number { const count = Number(value); - if (!Number.isSafeInteger(count) || count < 0) throw new Error(`Invalid SQLite ${label}.`); + if (!Number.isSafeInteger(count) || count < 0) throw new CodeGraphStorageOperationError(`Invalid SQLite ${label}.`); return count; } function safeProduct(left: number, right: number, label: string): number { const value = left * right; - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid SQLite ${label}.`); + if (!Number.isSafeInteger(value) || value < 0) throw new CodeGraphStorageOperationError(`Invalid SQLite ${label}.`); return value; } @@ -724,7 +775,8 @@ function requireRegularFileIdentity(fs: FileSystem.FileSystem, candidate: string return fs.stat(candidate).pipe( Effect.flatMap(info => Option.match(fileIdentity(info), { - onNone: () => Effect.fail(new Error('Code graph database lacks stable identity metadata.')), + onNone: () => + Effect.fail(new CodeGraphStorageOperationError('Code graph database lacks stable identity metadata.')), onSome: Effect.succeed, }), ), @@ -734,12 +786,16 @@ function requireRegularFileIdentity(fs: FileSystem.FileSystem, candidate: string function verifyRegularFileIdentity(fs: FileSystem.FileSystem, candidate: string, expected: CodeGraphFileIdentity) { return Effect.gen(function* () { if (Option.isSome(yield* fs.readLink(candidate).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph database became a symbolic link before compaction.')); + return yield* Effect.fail( + new CodeGraphStorageOperationError('Code graph database became a symbolic link before compaction.'), + ); } const current = yield* requireRegularFileIdentity(fs, candidate); if (!sameFileIdentity(expected, current)) { return yield* Effect.fail( - new Error('Code graph database changed before compaction; retry after current work finishes.'), + new CodeGraphStorageOperationError( + 'Code graph database changed before compaction; retry after current work finishes.', + ), ); } }); diff --git a/src/code_graph/store_health.ts b/src/code_graph/store_health.ts index 0ac3f4a0..4dee9a90 100644 --- a/src/code_graph/store_health.ts +++ b/src/code_graph/store_health.ts @@ -1,5 +1,5 @@ import * as SqliteClient from '@effect/sql-sqlite-bun/SqliteClient'; -import {Effect} from 'effect'; +import {Effect, Layer} from 'effect'; import * as SqlClient from 'effect/unstable/sql/SqlClient'; import {type CodeGraphDatabaseHealth} from './store_models.js'; import {codeGraphPersistentExtensionSchemaCompatible} from './store_schema_inspection.js'; @@ -40,67 +40,66 @@ export const diagnoseCodeGraphDatabaseReadOnly = Effect.fn('codeGraph.diagnoseDa databasePath: string, deep: boolean, ) { - return yield* Effect.scoped( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql.unsafe('PRAGMA query_only = ON'); - yield* sql.unsafe(`PRAGMA busy_timeout = ${deep ? 5_000 : 250}`); - const integrityRows = deep - ? yield* sql.unsafe<{readonly integrity_check: string}>('PRAGMA integrity_check(10)') - : [{integrity_check: 'ok'}]; - const schemaRows = yield* sql<{readonly value: string}>` + const inspect = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql.unsafe('PRAGMA query_only = ON'); + yield* sql.unsafe(`PRAGMA busy_timeout = ${deep ? 5_000 : 250}`); + const integrityRows = deep + ? yield* sql.unsafe<{readonly integrity_check: string}>('PRAGMA integrity_check(10)') + : [{integrity_check: 'ok'}]; + const schemaRows = yield* sql<{readonly value: string}>` SELECT value FROM schema_metadata WHERE key = 'schema_version' `; - const schemaVersion = Number.parseInt(schemaRows[0]?.value ?? '', 10); - const cleanupAdmission = yield* codeGraphRemovedViewCleanupSchemaAdmission(sql); - const persistentExtensionSchemaRevision = cleanupAdmission.persistentExtensionSchemaRevision; - const persistentExtensionCurrent = - persistentExtensionSchemaRevision === CODE_GRAPH_PERSISTENT_EXTENSION_SCHEMA_REVISION && - (yield* codeGraphPersistentExtensionSchemaCompatible(sql)) && - cleanupAdmission.current; - const coreReadSchemaCompatible = yield* codeGraphWorktreeReconciliationSchemaCompatible( - sql, - false, - false, - false, - false, - ); - const stateRows = yield* sql<{readonly count: number; readonly state: CodeGraphSnapshot['state']}>` + const schemaVersion = Number.parseInt(schemaRows[0]?.value ?? '', 10); + const cleanupAdmission = yield* codeGraphRemovedViewCleanupSchemaAdmission(sql); + const persistentExtensionSchemaRevision = cleanupAdmission.persistentExtensionSchemaRevision; + const persistentExtensionCurrent = + persistentExtensionSchemaRevision === CODE_GRAPH_PERSISTENT_EXTENSION_SCHEMA_REVISION && + (yield* codeGraphPersistentExtensionSchemaCompatible(sql)) && + cleanupAdmission.current; + const coreReadSchemaCompatible = yield* codeGraphWorktreeReconciliationSchemaCompatible( + sql, + false, + false, + false, + false, + ); + const stateRows = yield* sql<{readonly count: number; readonly state: CodeGraphSnapshot['state']}>` SELECT state, COUNT(*) AS count FROM snapshots GROUP BY state `; - const activeRows = yield* sql<{readonly count: number}>`SELECT COUNT(*) AS count FROM active_snapshots`; - const cacheRows = deep ? yield* sql<{readonly count: number}>`SELECT COUNT(*) AS count FROM file_blobs` : []; - const foreignKeyRows = deep ? yield* sql.unsafe('PRAGMA foreign_key_check') : []; - const counts = new Map(stateRows.map(row => [row.state, Number(row.count)])); - const integrityOk = - integrityRows.length === 1 && integrityRows[0]?.integrity_check === 'ok' && foreignKeyRows.length === 0; - return { - activeSnapshots: Number(activeRows[0]?.count ?? 0), - buildingSnapshots: counts.get('building') ?? 0, - cachedFileBlobs: Number(cacheRows[0]?.count ?? 0), - failedSnapshots: counts.get('failed') ?? 0, - foreignKeyViolations: foreignKeyRows.length, - integrity: codeGraphDatabaseIntegrity({ - coreReadSchemaCompatible, - integrityOk, - persistentExtensionCurrent, - persistentExtensionSchemaRevision, - schemaVersion: Number.isSafeInteger(schemaVersion) ? schemaVersion : undefined, - }), - readySnapshots: counts.get('ready') ?? 0, + const activeRows = yield* sql<{readonly count: number}>`SELECT COUNT(*) AS count FROM active_snapshots`; + const cacheRows = deep ? yield* sql<{readonly count: number}>`SELECT COUNT(*) AS count FROM file_blobs` : []; + const foreignKeyRows = deep ? yield* sql.unsafe('PRAGMA foreign_key_check') : []; + const counts = new Map(stateRows.map(row => [row.state, Number(row.count)])); + const integrityOk = + integrityRows.length === 1 && integrityRows[0]?.integrity_check === 'ok' && foreignKeyRows.length === 0; + return { + activeSnapshots: Number(activeRows[0]?.count ?? 0), + buildingSnapshots: counts.get('building') ?? 0, + cachedFileBlobs: Number(cacheRows[0]?.count ?? 0), + failedSnapshots: counts.get('failed') ?? 0, + foreignKeyViolations: foreignKeyRows.length, + integrity: codeGraphDatabaseIntegrity({ + coreReadSchemaCompatible, + integrityOk, + persistentExtensionCurrent, persistentExtensionSchemaRevision, schemaVersion: Number.isSafeInteger(schemaVersion) ? schemaVersion : undefined, - } satisfies CodeGraphDatabaseHealth; - }).pipe( - Effect.provide( - SqliteClient.layer({ - create: false, - disableWAL: true, - filename: databasePath, - readonly: true, - readwrite: false, - }), - ), - ), + }), + readySnapshots: counts.get('ready') ?? 0, + persistentExtensionSchemaRevision, + schemaVersion: Number.isSafeInteger(schemaVersion) ? schemaVersion : undefined, + } satisfies CodeGraphDatabaseHealth; + }); + return yield* Effect.scoped( + Layer.build( + SqliteClient.layer({ + create: false, + disableWAL: true, + filename: databasePath, + readonly: true, + readwrite: false, + }), + ).pipe(Effect.flatMap(context => inspect.pipe(Effect.provide(context)))), ); }); diff --git a/src/code_graph/store_queries.ts b/src/code_graph/store_queries.ts index 8817c4fc..3d891e25 100644 --- a/src/code_graph/store_queries.ts +++ b/src/code_graph/store_queries.ts @@ -487,16 +487,13 @@ const selectCachedFacts = Effect.fn('codeGraph.selectCachedFacts')(function* ( } const rows = yield* selectFileBlobBatch(sql, batch, extractorSet); for (const row of rows) { - try { - const bounded = decodeStoredCodeGraphFact(row.facts_json, row.path_hint); - output.set(row.path_hint, bounded.facts); - keys.add(row.path_hint); - const factBytes = bounded.bytes; - bytes += factBytes; - bytesByPath.set(row.path_hint, factBytes); - } catch { - // A malformed cache row is disposable and will be replaced after extraction. - } + const bounded = decodeStoredCodeGraphFactOption(row.facts_json, row.path_hint); + if (bounded === undefined) continue; + output.set(row.path_hint, bounded.facts); + keys.add(row.path_hint); + const factBytes = bounded.bytes; + bytes += factBytes; + bytesByPath.set(row.path_hint, factBytes); } const missing = batch.filter(file => !keys.has(file.path)); const reusableRows = yield* selectReusableFileBlobBatch(sql, missing, extractorSet); @@ -505,18 +502,12 @@ const selectCachedFacts = Effect.fn('codeGraph.selectCachedFacts')(function* ( if (keys.has(row.target_path)) continue; const file = filesByPath.get(row.target_path); if (file === undefined) continue; - try { - const bounded = decodeStoredCodeGraphFact(row.facts_json, row.path_hint); - const relocated = relocateStructuredSchemaFacts(file, bounded.facts); - if (relocated === undefined) continue; - const factBytes = codeGraphUtf8ByteLength(JSON.stringify(relocated)); - output.set(row.target_path, relocated); - keys.add(row.target_path); - bytes += factBytes; - bytesByPath.set(row.target_path, factBytes); - } catch { - // A malformed or incompatible donor is disposable and cannot satisfy this target path. - } + const relocated = relocateStoredCodeGraphFactOption(row.facts_json, row.path_hint, file); + if (relocated === undefined) continue; + output.set(row.target_path, relocated.facts); + keys.add(row.target_path); + bytes += relocated.bytes; + bytesByPath.set(row.target_path, relocated.bytes); } } return {bytes, bytesByPath, facts: output, keys} satisfies LoadedCodeGraphFacts; @@ -549,27 +540,48 @@ const selectMaterializedFileShards = Effect.fn('codeGraph.selectMaterializedFile [extractorSet, derivationIdentity, ...batch.flatMap(file => [file.contentHash, file.path])], ); for (const row of rows) { - try { - const bounded = decodeStoredCodeGraphFact(row.facts_json, row.path_hint); - if ( - bounded.facts.path !== row.path_hint || - row.id !== materializedFileShardIdentity(row.content_hash, extractorSet, derivationIdentity, row.path_hint) - ) { - continue; - } - output.set(row.path_hint, bounded.facts); - keys.add(row.path_hint); - const factBytes = bounded.bytes; - bytes += factBytes; - bytesByPath.set(row.path_hint, factBytes); - } catch { - // Materialized shards are disposable; malformed rows are ignored and rebuilt. + const bounded = decodeStoredCodeGraphFactOption(row.facts_json, row.path_hint); + if ( + bounded === undefined || + bounded.facts.path !== row.path_hint || + row.id !== materializedFileShardIdentity(row.content_hash, extractorSet, derivationIdentity, row.path_hint) + ) { + continue; } + output.set(row.path_hint, bounded.facts); + keys.add(row.path_hint); + const factBytes = bounded.bytes; + bytes += factBytes; + bytesByPath.set(row.path_hint, factBytes); } } return {bytes, bytesByPath, facts: output, keys} satisfies LoadedCodeGraphFacts; }); +function decodeStoredCodeGraphFactOption(json: string, path: string) { + try { + return decodeStoredCodeGraphFact(json, path); + } catch { + // Cached graph facts are disposable; malformed rows are ignored and rebuilt. + return undefined; + } +} + +function relocateStoredCodeGraphFactOption( + json: string, + sourcePath: string, + file: Pick, +): {readonly bytes: number; readonly facts: CodeGraphFileFacts} | undefined { + try { + const bounded = decodeStoredCodeGraphFact(json, sourcePath); + const facts = relocateStructuredSchemaFacts(file, bounded.facts); + return facts === undefined ? undefined : {bytes: codeGraphUtf8ByteLength(JSON.stringify(facts)), facts}; + } catch { + // Malformed or incompatible donors cannot satisfy the target path. + return undefined; + } +} + function selectFileBlobBatch(sql: SqlClient.SqlClient, files: readonly CodeGraphBlobReuseFile[], extractorSet: string) { if (files.length === 0) { return Effect.succeed([] as readonly (FileBlobRow & {readonly facts_bytes: number; readonly path_hint: string})[]); @@ -682,11 +694,7 @@ interface SearchSymbolRow extends SymbolRow { readonly score: number; } -/** - * Build exact-match candidates with the equality predicate inside every - * current/base branch. Keeping the predicate outside effectiveSymbolsCte() - * makes SQLite scan every symbol in a large snapshot before applying LIMIT. - */ +/** Keep exact predicates inside each branch so SQLite does not scan every symbol before LIMIT. */ function compactLexicalTermBranch(alias: string, placeholders: string, base: boolean): string { const suppression = base diff --git a/src/code_graph/store_session.ts b/src/code_graph/store_session.ts index e5080c51..1ec63c6a 100644 --- a/src/code_graph/store_session.ts +++ b/src/code_graph/store_session.ts @@ -1,5 +1,5 @@ import * as SqliteClient from '@effect/sql-sqlite-bun/SqliteClient'; -import {Context, Effect, Option, Path} from 'effect'; +import {Context, Effect, Layer, Option, Path} from 'effect'; import * as SqlClient from 'effect/unstable/sql/SqlClient'; import * as SqlError from 'effect/unstable/sql/SqlError'; import type { @@ -56,18 +56,16 @@ export function useExistingDatabase( effect: Effect.Effect, ): Effect.Effect> { return Effect.scoped( - effect.pipe( - Effect.provide( - SqliteClient.layer({ - create: false, - disableWAL: true, - filename: databasePath, - readonly: false, - readwrite: true, - }), - ), - ), - ) as Effect.Effect>; + Layer.build( + SqliteClient.layer({ + create: false, + disableWAL: true, + filename: databasePath, + readonly: false, + readwrite: true, + }), + ).pipe(Effect.flatMap(context => effect.pipe(Effect.provide(context)))), + ); } export function useDatabaseDirect( @@ -84,7 +82,7 @@ export function useDatabaseDirect( readwrite: false, }) : SqliteClient.layer({disableWAL: true, filename: databasePath}); - return Effect.scoped(effect.pipe(Effect.provide(layer))) as Effect.Effect>; + return Effect.scoped(Layer.build(layer).pipe(Effect.flatMap(context => effect.pipe(Effect.provide(context))))); } export const configureConnection = Effect.fn('codeGraph.configureConnection')(function* (sql: SqlClient.SqlClient) { diff --git a/src/code_graph/tree_sitter/runtime.ts b/src/code_graph/tree_sitter/runtime.ts index 2b661a36..3efe5c15 100644 --- a/src/code_graph/tree_sitter/runtime.ts +++ b/src/code_graph/tree_sitter/runtime.ts @@ -1,7 +1,7 @@ import {Context, Effect, Exit, FileSystem, Layer, Option, Path, Semaphore} from 'effect'; import {Language, Parser, type Node} from 'web-tree-sitter'; import {sha256HexSync} from '../../crypto/sha256.js'; -import {fromPromiseError} from '../../effect/errors.js'; +import {fromPromise} from '../../effect/errors.js'; import {SystemInfo} from '../../effect/system.js'; import {toolRoot} from '../../utils.js'; import type {VerifiedLanguageAsset} from '../languages/types.js'; @@ -38,7 +38,9 @@ export class TreeSitterRuntime extends Context.Service Parser.init({locateFile: () => runtimePath}))), + Effect.andThen( + fromPromise('initialize tree-sitter parser', () => Parser.init({locateFile: () => runtimePath})), + ), Effect.mapError(cause => cause instanceof TreeSitterRuntimeError ? cause @@ -54,7 +56,7 @@ export class TreeSitterRuntime extends Context.Service Language.load(languagePath))), + Effect.andThen(fromPromise('load tree-sitter language', () => Language.load(languagePath))), Effect.mapError(cause => cause instanceof TreeSitterRuntimeError ? cause @@ -80,11 +82,11 @@ export class TreeSitterRuntime extends Context.Service + Effect.flatMap(({loading}) => loading.pipe( Effect.onExit(exit => Exit.isFailure(exit) diff --git a/src/code_graph/types.ts b/src/code_graph/types.ts index b0b7db6b..7f41e8dd 100644 --- a/src/code_graph/types.ts +++ b/src/code_graph/types.ts @@ -29,6 +29,8 @@ export type CodeGraphRelation = | 'tests'; export interface RepositoryIdentity { + /** Local-only current branch when HEAD is attached. */ + readonly branch?: string; readonly caseMode: 'insensitive' | 'sensitive'; readonly checkoutId: string; readonly displayName: string; diff --git a/src/code_graph/vector_maintenance.ts b/src/code_graph/vector_maintenance.ts index 110644a1..1bd63399 100644 --- a/src/code_graph/vector_maintenance.ts +++ b/src/code_graph/vector_maintenance.ts @@ -1,9 +1,26 @@ -import {Crypto, Effect, Encoding, FileSystem, Option, Path, PlatformError, Result} from 'effect'; +import {Crypto, Effect, FileSystem, Option, Path, PlatformError} from 'effect'; import {sha256HexSync} from '../crypto/sha256.js'; import {isFileLockTimeout, withExclusiveFileLock} from '../effect/file_lock.js'; import {runtimeTextDirectoryNamePage, SystemInfo, type SystemInfoShape} from '../effect/system.js'; import type {CodeGraphDirectPersistentCapacityBoundary} from './disk_capacity.js'; import {codeGraphVectorRetirementCursorLockPath, codeGraphVectorWriteLockPath} from './layout.js'; +import { + HASH_ID, + MODEL_ID, + ORDINARY_VECTOR_CURSOR_LIMIT, + VECTOR_DATABASE_LIMIT, + clearOrdinaryVectorRoundFlags, + encodeOrdinaryVectorPhaseCursor, + initialOrdinaryVectorCursor, + ordinaryVectorAdmissionCursor, + ordinaryVectorMarkerCursor, + parseOrdinaryVectorPhaseCursor, + restartOrdinaryVectorRound, + setOrdinaryVectorModelCursor, + updateOrdinaryVectorCursor, + type OrdinaryVectorModelCursor, + type OrdinaryVectorPhaseCursor, +} from './vector_maintenance_cursor.js'; import { type CodeGraphVectorPageStorage, commitCodeGraphVectorRetirementAdmission, @@ -57,22 +74,20 @@ export { selectCodeGraphVectorRetirementMarkerCandidate, } from './vector_retirement.js'; +class CodeGraphVectorMaintenanceError extends Error { + readonly _tag = 'CodeGraphVectorMaintenanceError' as const; +} + const VECTOR_DATABASE_VERSION = 2; const VECTOR_DATABASE_NAME = `vectors-v${VECTOR_DATABASE_VERSION}.sqlite`; -const VECTOR_DATABASE_LIMIT = 64; const VECTOR_DIRECTORY_ENTRY_LIMIT = 66; -const MODEL_ID = /^[a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?$/; -const HASH_ID = /^[0-9a-f]{64}$/; const VECTOR_PHASE_CURSOR = /^vp1:(r|n|a):([0-9a-f]{64})(?::([a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?))?(?::([0-9]+))?$/u; -const ORDINARY_VECTOR_CURSOR = /^ov1:([0-9a-f]{64}):([-_A-Za-z0-9]+)$/u; -const ORDINARY_VECTOR_CURSOR_LIMIT = 64 * 1_024; const ORDINARY_VECTOR_CURSOR_FILE = '.ordinary-vector-retirement-v1.cursor'; const ORDINARY_VECTOR_CURSOR_TEMPORARY = '.ordinary-vector-retirement-v1.cursor.tmp'; const ORDINARY_VECTOR_CURSOR_METADATA_BYTES = 1_024; const VECTOR_UNIT_RETRY_MILLISECONDS = 1_000; const VECTOR_UNIT_INVALID_RETRY_MILLISECONDS = 30_000; -const ORDINARY_VECTOR_GENERATION_BYTES = 256; export const CODE_GRAPH_ORDINARY_VECTOR_UNIT_DEADLINE_MILLISECONDS = 250; export type CodeGraphVectorCleanupWarningCode = @@ -135,20 +150,6 @@ type VectorPhaseCursor = | {readonly digest: string; readonly mode: 'next'; readonly modelName: string} | {readonly digest: string; readonly mode: 'active'; readonly modelName: string; readonly step: number}; -interface OrdinaryVectorModelCursor { - readonly admissionWrapped: boolean; - readonly afterGeneration: string; - readonly phase: 'admission' | 'marker' | 'verified'; -} - -interface OrdinaryVectorPhaseCursor { - readonly digest: string; - readonly models: ReadonlyMap; - readonly nextModelName?: string; - readonly roundDeferred: boolean; - readonly roundProgressed: boolean; -} - export interface CodeGraphRemovedViewVectorUnitInput { readonly checkoutId: string; readonly threadnoteHome: string; @@ -591,7 +592,9 @@ function runCodeGraphOrdinaryVectorMaintenanceWithCursor( directoryInfo.dev !== databaseInfo.dev || !sameVectorDatabaseFileIdentity(candidate.fileIdentity, vectorDatabaseFileIdentity(databaseInfo)) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor filesystem changed.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor filesystem changed.'), + ); } yield* writeOrdinaryVectorCursorCas(fs, path, crypto, authority, cursorToken, intentToken); intentPublished = true; @@ -673,7 +676,9 @@ function runCodeGraphOrdinaryVectorMaintenanceWithCursor( yield* inspectCodeGraphVectorPageStorage(candidate.databasePath), ) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector checkpoint authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector checkpoint authority changed.'), + ); } if (!verifyCompletion) { yield* writeOrdinaryVectorCursorCas(fs, path, crypto, authority, cursorToken, nextToken); @@ -694,7 +699,9 @@ function runCodeGraphOrdinaryVectorMaintenanceWithCursor( vectorInventoryDigest(before.candidates) !== cursor.digest || !sameVectorDatabaseInventory(inventory, before) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector inventory changed.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector inventory changed.'), + ); } let observedDirty = false; for (const lockedCandidate of before.candidates) { @@ -709,7 +716,9 @@ function runCodeGraphOrdinaryVectorMaintenanceWithCursor( vectorInventoryDigest(after.candidates) !== cursor.digest || !sameVectorDatabaseInventory(before, after) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector inventory changed.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector inventory changed.'), + ); } if (preparation.afterFinalVerificationBeforeCursorCas !== undefined) { yield* preparation.afterFinalVerificationBeforeCursorCas(); @@ -819,37 +828,6 @@ function runCodeGraphOrdinaryVectorMaintenanceWithCursor( }); } -function ordinaryVectorAdmissionCursor(afterGeneration = ''): OrdinaryVectorModelCursor { - return {admissionWrapped: false, afterGeneration, phase: 'admission'}; -} - -function ordinaryVectorMarkerCursor(afterGeneration: string, admissionWrapped: boolean): OrdinaryVectorModelCursor { - return {admissionWrapped, afterGeneration, phase: 'marker'}; -} - -function initialOrdinaryVectorCursor(digest: string): OrdinaryVectorPhaseCursor { - return {digest, models: new Map(), roundDeferred: false, roundProgressed: false}; -} - -function restartOrdinaryVectorRound(cursor: OrdinaryVectorPhaseCursor): OrdinaryVectorPhaseCursor { - return { - digest: cursor.digest, - models: cursor.models, - roundDeferred: false, - roundProgressed: false, - }; -} - -function clearOrdinaryVectorRoundFlags(cursor: OrdinaryVectorPhaseCursor): OrdinaryVectorPhaseCursor { - return { - digest: cursor.digest, - models: cursor.models, - ...(cursor.nextModelName === undefined ? {} : {nextModelName: cursor.nextModelName}), - roundDeferred: false, - roundProgressed: false, - }; -} - function withAllOrdinaryVectorModelLocks( fs: FileSystem.FileSystem, path: Path.Path, @@ -876,36 +854,6 @@ function withAllOrdinaryVectorModelLocks( ); } -function updateOrdinaryVectorCursor( - cursor: OrdinaryVectorPhaseCursor, - modelName: string, - modelCursor: OrdinaryVectorModelCursor, - progressed: boolean, - deferred: boolean, -): OrdinaryVectorPhaseCursor { - return { - digest: cursor.digest, - models: setOrdinaryVectorModelCursor(cursor.models, modelName, modelCursor), - nextModelName: modelName, - roundDeferred: cursor.roundDeferred || deferred, - roundProgressed: cursor.roundProgressed || progressed, - }; -} - -function setOrdinaryVectorModelCursor( - current: ReadonlyMap, - modelName: string, - modelCursor: OrdinaryVectorModelCursor, -): Map { - const updated = new Map(current); - if (modelCursor.phase === 'admission' && !modelCursor.admissionWrapped && modelCursor.afterGeneration === '') { - updated.delete(modelName); - } else { - updated.set(modelName, modelCursor); - } - return updated; -} - function ordinaryVectorProgress(cursor: OrdinaryVectorPhaseCursor): CodeGraphOrdinaryVectorMaintenanceUnitResult { return { cursorToken: encodeOrdinaryVectorPhaseCursor(cursor), @@ -948,114 +896,11 @@ export function codeGraphOrdinaryVectorMaintenanceBoundary( finalFactBytes <= 0 || rowCount <= 0 ) { - throw new Error('Code graph ordinary vector capacity boundary is invalid.'); + throw new CodeGraphVectorMaintenanceError('Code graph ordinary vector capacity boundary is invalid.'); } return {finalFactBytes, operation: 'maintain code graph vector retirement', rowCount}; } -function encodeOrdinaryVectorPhaseCursor(cursor: OrdinaryVectorPhaseCursor): string { - const models = [...cursor.models.entries()] - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([modelName, state]) => [ - modelName, - state.phase === 'admission' ? 'a' : state.phase === 'marker' ? 'm' : 'v', - state.admissionWrapped ? 1 : 0, - state.afterGeneration, - ]); - const payload = Encoding.encodeBase64Url( - JSON.stringify({ - v: 1, - d: cursor.digest, - ...(cursor.nextModelName === undefined ? {} : {n: cursor.nextModelName}), - m: models, - p: cursor.roundProgressed ? 1 : 0, - x: cursor.roundDeferred ? 1 : 0, - }), - ); - const seal = sha256HexSync(`code-graph-ordinary-vector-cursor-v1\n${payload}`); - return `ov1:${seal}:${payload}`; -} - -function parseOrdinaryVectorPhaseCursor(cursorToken: string | undefined): OrdinaryVectorPhaseCursor | undefined { - if (cursorToken === undefined || cursorToken.length > ORDINARY_VECTOR_CURSOR_LIMIT) return undefined; - const match = ORDINARY_VECTOR_CURSOR.exec(cursorToken); - if (match === null) return undefined; - const [, seal, payload] = match; - if (sha256HexSync(`code-graph-ordinary-vector-cursor-v1\n${payload}`) !== seal) return undefined; - const decoded = Encoding.decodeBase64UrlString(payload!); - if (!Result.isSuccess(decoded) || Encoding.encodeBase64Url(decoded.success) !== payload) return undefined; - let raw: unknown; - try { - raw = JSON.parse(decoded.success); - } catch { - return undefined; - } - const cursor = decodeOrdinaryVectorCursorPayload(raw); - return cursor !== undefined && encodeOrdinaryVectorPhaseCursor(cursor) === cursorToken ? cursor : undefined; -} - -function decodeOrdinaryVectorCursorPayload(raw: unknown): OrdinaryVectorPhaseCursor | undefined { - if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return undefined; - const candidate = raw as { - readonly d?: unknown; - readonly m?: unknown; - readonly n?: unknown; - readonly p?: unknown; - readonly v?: unknown; - readonly x?: unknown; - }; - if ( - candidate.v !== 1 || - typeof candidate.d !== 'string' || - !HASH_ID.test(candidate.d) || - (candidate.p !== 0 && candidate.p !== 1) || - (candidate.x !== 0 && candidate.x !== 1) - ) { - return undefined; - } - if (candidate.n !== undefined && (typeof candidate.n !== 'string' || !MODEL_ID.test(candidate.n))) return undefined; - if (!Array.isArray(candidate.m) || candidate.m.length > VECTOR_DATABASE_LIMIT) return undefined; - const models = new Map(); - let previous = ''; - for (const entry of candidate.m) { - if (!Array.isArray(entry) || entry.length !== 4) return undefined; - const [modelName, phase, wrapped, afterGeneration] = entry; - if ( - typeof modelName !== 'string' || - !MODEL_ID.test(modelName) || - modelName <= previous || - (phase !== 'a' && phase !== 'm' && phase !== 'v') || - (wrapped !== 0 && wrapped !== 1) || - typeof afterGeneration !== 'string' || - (afterGeneration !== '' && !validOrdinaryVectorGeneration(afterGeneration)) || - (phase === 'a' && wrapped !== 0) || - (phase === 'v' && (wrapped !== 1 || afterGeneration !== '')) - ) { - return undefined; - } - const state: OrdinaryVectorModelCursor = { - admissionWrapped: wrapped === 1, - afterGeneration, - phase: phase === 'a' ? 'admission' : phase === 'm' ? 'marker' : 'verified', - }; - if (state.phase === 'admission' && state.afterGeneration === '') return undefined; - models.set(modelName, state); - previous = modelName; - } - return { - digest: candidate.d, - models, - ...(candidate.n === undefined ? {} : {nextModelName: candidate.n as string}), - roundDeferred: candidate.x === 1, - roundProgressed: candidate.p === 1, - }; -} - -function validOrdinaryVectorGeneration(generation: string): boolean { - const bytes = new TextEncoder().encode(generation).byteLength; - return bytes > 0 && bytes <= ORDINARY_VECTOR_GENERATION_BYTES && !generation.includes('\0'); -} - function ordinaryVectorCursorMatchesInventory( cursor: OrdinaryVectorPhaseCursor, inventory: VectorDatabaseInventory, @@ -1107,7 +952,9 @@ function inspectOrdinaryVectorCursorAuthority( const directory = vectorRoot; const info = yield* fs.stat(directory); if (info.type !== 'Directory' || (yield* fs.realPath(directory)) !== directory) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor directory is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor directory is invalid.'), + ); } const authority = { cursorPath: path.join(directory, ORDINARY_VECTOR_CURSOR_FILE), @@ -1119,7 +966,9 @@ function inspectOrdinaryVectorCursorAuthority( } satisfies OrdinaryVectorCursorAuthority; for (const target of [authority.cursorPath, authority.temporaryPath]) { if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor authority contains a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor authority contains a symbolic link.'), + ); } } return authority; @@ -1139,15 +988,21 @@ function revalidateOrdinaryVectorCursorAuthority( authority.vectorRoot !== authority.directory || !sameVectorDatabaseFileIdentity(authority.directoryIdentity, vectorDatabaseFileIdentity(info)) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor directory changed identity.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor directory changed identity.'), + ); } for (const target of [authority.cursorPath, authority.temporaryPath]) { if (Option.isSome(yield* fs.readLink(target).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor authority became a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor authority became a symbolic link.'), + ); } const targetInfo = yield* optionalVectorFileInfo(fs, target); if (Option.isSome(targetInfo) && targetInfo.value.type !== 'File') { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor authority changed type.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor authority changed type.'), + ); } } }); @@ -1163,7 +1018,9 @@ function recoverOrdinaryVectorCursorTemporary( const info = yield* optionalVectorFileInfo(fs, authority.temporaryPath); if (Option.isNone(info)) return; if (info.value.type !== 'File') { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor temporary is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor temporary is invalid.'), + ); } yield* removeOrdinaryVectorCursorFileIfOwned( fs, @@ -1218,13 +1075,15 @@ function writeOrdinaryVectorCursorCas( observed.state === 'invalid' || (observed.state === 'cursor' ? observed.cursorToken : undefined) !== expectedToken ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor CAS changed.')); + return yield* Effect.fail(new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor CAS changed.')); } if (nextToken === undefined) { if (observed.state === 'cursor') { const info = yield* fs.stat(authority.cursorPath); if (info.type !== 'File') { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor changed type.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor changed type.'), + ); } yield* removeOrdinaryVectorCursorFileIfOwned( fs, @@ -1242,14 +1101,18 @@ function writeOrdinaryVectorCursorCas( parseOrdinaryVectorPhaseCursor(nextToken) === undefined || new TextEncoder().encode(content).byteLength > ORDINARY_VECTOR_CURSOR_LIMIT + 1 ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor exceeded its exact bound.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor exceeded its exact bound.'), + ); } yield* recoverOrdinaryVectorCursorTemporary(fs, path, authority); yield* revalidateOrdinaryVectorCursorAuthority(fs, path, authority); yield* fs.writeFileString(authority.temporaryPath, content, {flag: 'wx', mode: 0o600}); const temporaryInfo = yield* fs.stat(authority.temporaryPath); if (temporaryInfo.type !== 'File') { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor temporary changed type.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor temporary changed type.'), + ); } temporaryIdentity = vectorDatabaseFileIdentity(temporaryInfo); yield* syncOrdinaryVectorFile(fs, authority.temporaryPath); @@ -1259,7 +1122,9 @@ function writeOrdinaryVectorCursorCas( reobserved.state === 'invalid' || (reobserved.state === 'cursor' ? reobserved.cursorToken : undefined) !== expectedToken ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor CAS changed before publication.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor CAS changed before publication.'), + ); } yield* revalidateOrdinaryVectorCursorAuthority(fs, path, authority); const finalTemporaryInfo = yield* fs.stat(authority.temporaryPath); @@ -1268,7 +1133,9 @@ function writeOrdinaryVectorCursorCas( temporaryIdentity === undefined || !sameVectorDatabaseFileIdentity(temporaryIdentity, vectorDatabaseFileIdentity(finalTemporaryInfo)) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor temporary changed identity.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor temporary changed identity.'), + ); } yield* fs.rename(authority.temporaryPath, authority.cursorPath); temporaryIdentity = undefined; @@ -1303,7 +1170,9 @@ function removeOrdinaryVectorCursorFileIfOwned( return Effect.gen(function* () { yield* revalidateOrdinaryVectorCursorAuthority(fs, path, authority); if (Option.isSome(yield* fs.readLink(file).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor file became a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor file became a symbolic link.'), + ); } const observed = yield* optionalVectorFileInfo(fs, file); if (Option.isNone(observed)) return; @@ -1311,11 +1180,15 @@ function removeOrdinaryVectorCursorFileIfOwned( observed.value.type !== 'File' || !sameVectorDatabaseFileIdentity(expectedIdentity, vectorDatabaseFileIdentity(observed.value)) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor file changed identity.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor file changed identity.'), + ); } yield* revalidateOrdinaryVectorCursorAuthority(fs, path, authority); if (Option.isSome(yield* fs.readLink(file).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor file became a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor file became a symbolic link.'), + ); } const confirmed = yield* optionalVectorFileInfo(fs, file); if ( @@ -1323,7 +1196,9 @@ function removeOrdinaryVectorCursorFileIfOwned( confirmed.value.type !== 'File' || !sameVectorDatabaseFileIdentity(expectedIdentity, vectorDatabaseFileIdentity(confirmed.value)) ) { - return yield* Effect.fail(new Error('Code graph ordinary vector cursor file changed before removal.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph ordinary vector cursor file changed before removal.'), + ); } yield* fs.remove(file, {force: false}); }); @@ -1357,14 +1232,14 @@ function ensureOrdinaryVectorUnitDeadline( return Effect.suspend(() => ordinaryVectorMonotonicMilliseconds(preparation) < preparation.deadlineMonotonicMilliseconds ? Effect.void - : Effect.fail(new Error('Code graph ordinary vector maintenance deadline expired.')), + : Effect.fail(new CodeGraphVectorMaintenanceError('Code graph ordinary vector maintenance deadline expired.')), ); } function ensureVectorUnitDeadline(preparation: CodeGraphRemovedViewVectorUnitPreparation): Effect.Effect { return Effect.suspend(() => (preparation.monotonicMilliseconds?.() ?? performance.now()) < preparation.deadlineMonotonicMilliseconds ? Effect.void - : Effect.fail(new Error('Code graph vector cleanup deadline expired.')), + : Effect.fail(new CodeGraphVectorMaintenanceError('Code graph vector cleanup deadline expired.')), ); } @@ -1616,7 +1491,9 @@ const validateSnapshotVectorTarget = Effect.fn('codeGraph.validateSnapshotVector snapshotId: string, ) { if (!HASH_ID.test(checkoutId) || !validSnapshotId(snapshotId)) { - return yield* Effect.fail(new Error('Code graph vector snapshot purge target is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector snapshot purge target is invalid.'), + ); } }); @@ -1724,7 +1601,7 @@ export const cleanupCodeGraphVectorPointers = Effect.fn('codeGraph.cleanupVector expectedSnapshotId: string, ) { if (!HASH_ID.test(checkoutId) || !HASH_ID.test(worktreeId) || !validSnapshotId(expectedSnapshotId)) { - return yield* Effect.fail(new Error('Code graph vector cleanup target is invalid.')); + return yield* Effect.fail(new CodeGraphVectorMaintenanceError('Code graph vector cleanup target is invalid.')); } const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -1866,25 +1743,32 @@ const inspectCanonicalVectorRoot = Effect.fn('codeGraph.inspectCanonicalVectorRo checkoutId: string, ) { if (Option.isSome(yield* fs.readLink(threadnoteHome).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Threadnote home is a symbolic link.')); + return yield* Effect.fail(new CodeGraphVectorMaintenanceError('Threadnote home is a symbolic link.')); } const homeInfo = yield* optionalVectorFileInfo(fs, threadnoteHome); if (Option.isNone(homeInfo)) return undefined; - if (homeInfo.value.type !== 'Directory') return yield* Effect.fail(new Error('Threadnote home is invalid.')); + if (homeInfo.value.type !== 'Directory') + return yield* Effect.fail(new CodeGraphVectorMaintenanceError('Threadnote home is invalid.')); let current = yield* fs.realPath(threadnoteHome); for (const segment of ['indexes', 'code-graph', 'repositories', checkoutId, 'vectors']) { const candidate = path.join(current, segment); if (Option.isSome(yield* fs.readLink(candidate).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph vector containment contains a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector containment contains a symbolic link.'), + ); } const info = yield* optionalVectorFileInfo(fs, candidate); if (Option.isNone(info)) return undefined; if (info.value.type !== 'Directory') { - return yield* Effect.fail(new Error('Code graph vector containment has an invalid entry type.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector containment has an invalid entry type.'), + ); } const canonical = yield* fs.realPath(candidate); if (canonical !== candidate || path.dirname(canonical) !== current || path.basename(canonical) !== segment) { - return yield* Effect.fail(new Error('Code graph vector root escaped its derived-store containment.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector root escaped its derived-store containment.'), + ); } current = canonical; } @@ -1901,17 +1785,21 @@ const validateVectorDatabaseCandidate = Effect.fn('codeGraph.validateVectorDatab Option.isSome(yield* fs.readLink(candidate.modelRoot).pipe(Effect.option)) || Option.isSome(yield* fs.readLink(candidate.databasePath).pipe(Effect.option)) ) { - return yield* Effect.fail(new Error('Code graph vector cleanup target became a symbolic link.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector cleanup target became a symbolic link.'), + ); } const [vectorInfo, modelInfo, databaseInfo] = yield* Effect.all( [fs.stat(candidate.vectorRoot), fs.stat(candidate.modelRoot), fs.stat(candidate.databasePath)], {concurrency: 1}, ); if (vectorInfo.type !== 'Directory' || modelInfo.type !== 'Directory' || databaseInfo.type !== 'File') { - return yield* Effect.fail(new Error('Code graph vector cleanup target changed type.')); + return yield* Effect.fail(new CodeGraphVectorMaintenanceError('Code graph vector cleanup target changed type.')); } if (!sameVectorDatabaseFileIdentity(candidate.fileIdentity, vectorDatabaseFileIdentity(databaseInfo))) { - return yield* Effect.fail(new Error('Code graph vector cleanup target changed identity.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector cleanup target changed identity.'), + ); } const [canonicalVectorRoot, canonicalModelRoot, canonicalDatabasePath] = yield* Effect.all( [fs.realPath(candidate.vectorRoot), fs.realPath(candidate.modelRoot), fs.realPath(candidate.databasePath)], @@ -1924,7 +1812,9 @@ const validateVectorDatabaseCandidate = Effect.fn('codeGraph.validateVectorDatab path.dirname(canonicalModelRoot) !== canonicalVectorRoot || path.dirname(canonicalDatabasePath) !== canonicalModelRoot ) { - return yield* Effect.fail(new Error('Code graph vector cleanup target escaped its derived-store root.')); + return yield* Effect.fail( + new CodeGraphVectorMaintenanceError('Code graph vector cleanup target escaped its derived-store root.'), + ); } }); diff --git a/src/code_graph/vector_maintenance_cursor.ts b/src/code_graph/vector_maintenance_cursor.ts new file mode 100644 index 00000000..e69e6c65 --- /dev/null +++ b/src/code_graph/vector_maintenance_cursor.ts @@ -0,0 +1,190 @@ +import {Encoding, Result} from 'effect'; +import {sha256HexSync} from '../crypto/sha256.js'; + +export const VECTOR_DATABASE_LIMIT = 64; +export const MODEL_ID = /^[a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?$/; +export const HASH_ID = /^[0-9a-f]{64}$/; +const ORDINARY_VECTOR_CURSOR = /^ov1:([0-9a-f]{64}):([-_A-Za-z0-9]+)$/u; +export const ORDINARY_VECTOR_CURSOR_LIMIT = 64 * 1_024; +const ORDINARY_VECTOR_GENERATION_BYTES = 256; + +export interface OrdinaryVectorModelCursor { + readonly admissionWrapped: boolean; + readonly afterGeneration: string; + readonly phase: 'admission' | 'marker' | 'verified'; +} + +export interface OrdinaryVectorPhaseCursor { + readonly digest: string; + readonly models: ReadonlyMap; + readonly nextModelName?: string; + readonly roundDeferred: boolean; + readonly roundProgressed: boolean; +} + +export function ordinaryVectorAdmissionCursor(afterGeneration = ''): OrdinaryVectorModelCursor { + return {admissionWrapped: false, afterGeneration, phase: 'admission'}; +} + +export function ordinaryVectorMarkerCursor( + afterGeneration: string, + admissionWrapped: boolean, +): OrdinaryVectorModelCursor { + return {admissionWrapped, afterGeneration, phase: 'marker'}; +} + +export function initialOrdinaryVectorCursor(digest: string): OrdinaryVectorPhaseCursor { + return {digest, models: new Map(), roundDeferred: false, roundProgressed: false}; +} + +export function restartOrdinaryVectorRound(cursor: OrdinaryVectorPhaseCursor): OrdinaryVectorPhaseCursor { + return { + digest: cursor.digest, + models: cursor.models, + roundDeferred: false, + roundProgressed: false, + }; +} + +export function clearOrdinaryVectorRoundFlags(cursor: OrdinaryVectorPhaseCursor): OrdinaryVectorPhaseCursor { + return { + digest: cursor.digest, + models: cursor.models, + ...(cursor.nextModelName === undefined ? {} : {nextModelName: cursor.nextModelName}), + roundDeferred: false, + roundProgressed: false, + }; +} + +export function updateOrdinaryVectorCursor( + cursor: OrdinaryVectorPhaseCursor, + modelName: string, + modelCursor: OrdinaryVectorModelCursor, + progressed: boolean, + deferred: boolean, +): OrdinaryVectorPhaseCursor { + return { + digest: cursor.digest, + models: setOrdinaryVectorModelCursor(cursor.models, modelName, modelCursor), + nextModelName: modelName, + roundDeferred: cursor.roundDeferred || deferred, + roundProgressed: cursor.roundProgressed || progressed, + }; +} + +export function setOrdinaryVectorModelCursor( + current: ReadonlyMap, + modelName: string, + modelCursor: OrdinaryVectorModelCursor, +): Map { + const updated = new Map(current); + if (modelCursor.phase === 'admission' && !modelCursor.admissionWrapped && modelCursor.afterGeneration === '') { + updated.delete(modelName); + } else { + updated.set(modelName, modelCursor); + } + return updated; +} + +export function encodeOrdinaryVectorPhaseCursor(cursor: OrdinaryVectorPhaseCursor): string { + const models = [...cursor.models.entries()] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([modelName, state]) => [ + modelName, + state.phase === 'admission' ? 'a' : state.phase === 'marker' ? 'm' : 'v', + state.admissionWrapped ? 1 : 0, + state.afterGeneration, + ]); + const payload = Encoding.encodeBase64Url( + JSON.stringify({ + v: 1, + d: cursor.digest, + ...(cursor.nextModelName === undefined ? {} : {n: cursor.nextModelName}), + m: models, + p: cursor.roundProgressed ? 1 : 0, + x: cursor.roundDeferred ? 1 : 0, + }), + ); + const seal = sha256HexSync(`code-graph-ordinary-vector-cursor-v1\n${payload}`); + return `ov1:${seal}:${payload}`; +} + +export function parseOrdinaryVectorPhaseCursor(cursorToken: string | undefined): OrdinaryVectorPhaseCursor | undefined { + if (cursorToken === undefined || cursorToken.length > ORDINARY_VECTOR_CURSOR_LIMIT) return undefined; + const match = ORDINARY_VECTOR_CURSOR.exec(cursorToken); + if (match === null) return undefined; + const [, seal, payload] = match; + if (sha256HexSync(`code-graph-ordinary-vector-cursor-v1\n${payload}`) !== seal) return undefined; + const decoded = Encoding.decodeBase64UrlString(payload!); + if (!Result.isSuccess(decoded) || Encoding.encodeBase64Url(decoded.success) !== payload) return undefined; + let raw: unknown; + try { + raw = JSON.parse(decoded.success); + } catch { + return undefined; + } + const cursor = decodeOrdinaryVectorCursorPayload(raw); + return cursor !== undefined && encodeOrdinaryVectorPhaseCursor(cursor) === cursorToken ? cursor : undefined; +} + +function decodeOrdinaryVectorCursorPayload(raw: unknown): OrdinaryVectorPhaseCursor | undefined { + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return undefined; + const candidate = raw as { + readonly d?: unknown; + readonly m?: unknown; + readonly n?: unknown; + readonly p?: unknown; + readonly v?: unknown; + readonly x?: unknown; + }; + if ( + candidate.v !== 1 || + typeof candidate.d !== 'string' || + !HASH_ID.test(candidate.d) || + (candidate.p !== 0 && candidate.p !== 1) || + (candidate.x !== 0 && candidate.x !== 1) + ) { + return undefined; + } + if (candidate.n !== undefined && (typeof candidate.n !== 'string' || !MODEL_ID.test(candidate.n))) return undefined; + if (!Array.isArray(candidate.m) || candidate.m.length > VECTOR_DATABASE_LIMIT) return undefined; + const models = new Map(); + let previous = ''; + for (const entry of candidate.m) { + if (!Array.isArray(entry) || entry.length !== 4) return undefined; + const [modelName, phase, wrapped, afterGeneration] = entry; + if ( + typeof modelName !== 'string' || + !MODEL_ID.test(modelName) || + modelName <= previous || + (phase !== 'a' && phase !== 'm' && phase !== 'v') || + (wrapped !== 0 && wrapped !== 1) || + typeof afterGeneration !== 'string' || + (afterGeneration !== '' && !validOrdinaryVectorGeneration(afterGeneration)) || + (phase === 'a' && wrapped !== 0) || + (phase === 'v' && (wrapped !== 1 || afterGeneration !== '')) + ) { + return undefined; + } + const state: OrdinaryVectorModelCursor = { + admissionWrapped: wrapped === 1, + afterGeneration, + phase: phase === 'a' ? 'admission' : phase === 'm' ? 'marker' : 'verified', + }; + if (state.phase === 'admission' && state.afterGeneration === '') return undefined; + models.set(modelName, state); + previous = modelName; + } + return { + digest: candidate.d, + models, + ...(candidate.n === undefined ? {} : {nextModelName: candidate.n as string}), + roundDeferred: candidate.x === 1, + roundProgressed: candidate.p === 1, + }; +} + +function validOrdinaryVectorGeneration(generation: string): boolean { + const bytes = new TextEncoder().encode(generation).byteLength; + return bytes > 0 && bytes <= ORDINARY_VECTOR_GENERATION_BYTES && !generation.includes('\0'); +} diff --git a/src/code_graph/vector_retirement.ts b/src/code_graph/vector_retirement.ts index 9f734a51..34a5087a 100644 --- a/src/code_graph/vector_retirement.ts +++ b/src/code_graph/vector_retirement.ts @@ -1,4 +1,3 @@ -import * as SqliteClient from '@effect/sql-sqlite-bun/SqliteClient'; import {Crypto, Effect, FileSystem, Option, Path} from 'effect'; import * as SqlClient from 'effect/unstable/sql/SqlClient'; import {sha256HexSync} from '../crypto/sha256.js'; @@ -10,646 +9,75 @@ import { import {codeGraphDiskReservationFilesystemKey, withCodeGraphDiskReservation} from './disk_reservation.js'; import {codeGraphDiskReservationLockPath, codeGraphDiskReservationRoot} from './layout.js'; import {classifyCodeGraphLifecycle} from './lifecycle_classification.js'; - -export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS = 1_000; -export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_BYTES = 32 * 1_024 * 1_024; -export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_FIXED_ROWS = 5; -export const CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS = 8_192; -export const CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES = 4 * 1_024 * 1_024; - -const MAXIMUM_SAFE_INTEGER_SQL = '9007199254740991'; -const VECTOR_GENERATION_BYTES = 256; -const VECTOR_SNAPSHOT_BYTES = 1_024; -const VECTOR_MODEL_ID_BYTES = 256; -const VECTOR_MODEL_SHA256_BYTES = 64; -const VECTOR_CREATED_AT_BYTES = 64; -const VECTOR_SYMBOL_BYTES = 1_024; -const VECTOR_FINGERPRINT_BYTES = 1_024; -const VECTOR_RETIREMENT_TRIGGER_SQL_BYTES = 65_536; -const VECTOR_CORE_TABLE_NAMES = ['vector_generations', 'vector_pointers', 'vectors'] as const; -const VECTOR_RETIREMENT_TABLE_NAMES = ['vector_retirement_state', 'vector_generation_retirements'] as const; - -function vectorGenerationManifestPredicate(alias: string): string { - return `typeof(${alias}.generation) = 'text' - AND length(CAST(${alias}.generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(${alias}.generation, char(0)) = 0 - AND typeof(${alias}.snapshot_id) = 'text' - AND length(CAST(${alias}.snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} - AND instr(${alias}.snapshot_id, char(0)) = 0 - AND typeof(${alias}.model_id) = 'text' - AND length(CAST(${alias}.model_id AS BLOB)) BETWEEN 1 AND ${VECTOR_MODEL_ID_BYTES} - AND instr(${alias}.model_id, char(0)) = 0 - AND typeof(${alias}.model_sha256) = 'text' - AND length(CAST(${alias}.model_sha256 AS BLOB)) = ${VECTOR_MODEL_SHA256_BYTES} - AND ${alias}.model_sha256 NOT GLOB '*[^0-9a-f]*' - AND typeof(${alias}.dimensions) = 'integer' - AND ${alias}.dimensions BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND typeof(${alias}.template_version) = 'integer' - AND ${alias}.template_version BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND typeof(${alias}.count) = 'integer' - AND ${alias}.count BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND typeof(${alias}.state) = 'text' - AND ${alias}.state IN ('building', 'ready') - AND typeof(${alias}.created_at) = 'text' - AND length(CAST(${alias}.created_at AS BLOB)) BETWEEN 1 AND ${VECTOR_CREATED_AT_BYTES} - AND instr(${alias}.created_at, char(0)) = 0`; -} - -export const CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_generations ( - generation TEXT PRIMARY KEY, - snapshot_id TEXT NOT NULL, - model_id TEXT NOT NULL, - model_sha256 TEXT NOT NULL, - dimensions INTEGER NOT NULL CHECK(dimensions > 0), - template_version INTEGER NOT NULL, - count INTEGER NOT NULL CHECK(count >= 0), - state TEXT NOT NULL CHECK(state IN ('building', 'ready')), - created_at TEXT NOT NULL -)`; - -export const CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_pointers ( - worktree_id TEXT PRIMARY KEY, - generation TEXT NOT NULL REFERENCES vector_generations(generation) ON DELETE CASCADE -)`; - -export const CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL = - 'CREATE INDEX IF NOT EXISTS vector_pointer_generation_lookup ON vector_pointers (generation)'; - -export const CODE_GRAPH_VECTORS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vectors ( - generation TEXT NOT NULL REFERENCES vector_generations(generation) ON DELETE CASCADE, - symbol_id TEXT NOT NULL, - fingerprint TEXT NOT NULL, - vector BLOB NOT NULL, - PRIMARY KEY (generation, symbol_id) -) WITHOUT ROWID`; - -export const CODE_GRAPH_VECTOR_REUSE_INDEX_SQL = - 'CREATE INDEX IF NOT EXISTS vector_reuse_lookup ON vectors (generation, symbol_id, fingerprint)'; - -export const CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_retirement_state ( - singleton INTEGER PRIMARY KEY NOT NULL CHECK ( - typeof(singleton) = 'integer' AND singleton = 1 - ), - admission_cursor TEXT CHECK ( - admission_cursor IS NULL OR ( - typeof(admission_cursor) = 'text' - AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(admission_cursor, char(0)) = 0 - ) - ), - generation_revision INTEGER NOT NULL DEFAULT 0 CHECK ( - typeof(generation_revision) = 'integer' - AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - ), - admission_scan_revision INTEGER CHECK ( - admission_scan_revision IS NULL OR ( - typeof(admission_scan_revision) = 'integer' - AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - ) - ), - clean_generation_revision INTEGER CHECK ( - clean_generation_revision IS NULL OR ( - typeof(clean_generation_revision) = 'integer' - AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - ) - ), - pointer_delete_worktree_id TEXT, - pointer_delete_generation TEXT, - pointer_delete_snapshot_id TEXT, - CHECK ((admission_cursor IS NULL) = (admission_scan_revision IS NULL)), - CHECK (admission_scan_revision IS NULL OR admission_scan_revision <= generation_revision), - CHECK (clean_generation_revision IS NULL OR clean_generation_revision <= generation_revision), - CHECK ( - admission_scan_revision IS NULL - OR clean_generation_revision IS NULL - OR clean_generation_revision <= admission_scan_revision - ), - CHECK ( - clean_generation_revision IS NULL - OR clean_generation_revision < generation_revision - OR (admission_cursor IS NULL AND admission_scan_revision IS NULL) - ), - CHECK ( - ( - pointer_delete_worktree_id IS NULL - AND pointer_delete_generation IS NULL - AND pointer_delete_snapshot_id IS NULL - ) OR ( - typeof(pointer_delete_worktree_id) = 'text' - AND length(CAST(pointer_delete_worktree_id AS BLOB)) = 64 - AND pointer_delete_worktree_id NOT GLOB '*[^0-9a-f]*' - AND typeof(pointer_delete_generation) = 'text' - AND length(CAST(pointer_delete_generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(pointer_delete_generation, char(0)) = 0 - AND typeof(pointer_delete_snapshot_id) = 'text' - AND length(CAST(pointer_delete_snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} - AND instr(pointer_delete_snapshot_id, char(0)) = 0 - ) - ) -) WITHOUT ROWID`; - -export const CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_generation_retirements ( - retirement_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK ( - typeof(retirement_id) = 'integer' - AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} - ), - generation TEXT NOT NULL UNIQUE CHECK ( - typeof(generation) = 'text' - AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(generation, char(0)) = 0 - ), - snapshot_id TEXT NOT NULL CHECK ( - typeof(snapshot_id) = 'text' - AND length(CAST(snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} - AND instr(snapshot_id, char(0)) = 0 - ), - retired_by_worktree_id TEXT CHECK ( - retired_by_worktree_id IS NULL OR ( - typeof(retired_by_worktree_id) = 'text' - AND length(CAST(retired_by_worktree_id AS BLOB)) = 64 - AND retired_by_worktree_id NOT GLOB '*[^0-9a-f]*' - ) - ), - page_revision INTEGER NOT NULL DEFAULT 0 CHECK ( - typeof(page_revision) = 'integer' - AND page_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - ), - delete_authorized INTEGER NOT NULL DEFAULT 0 CHECK ( - typeof(delete_authorized) = 'integer' AND delete_authorized IN (0, 1) - ) -)`; - -export const CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL = `CREATE INDEX IF NOT EXISTS vector_generation_retirement_association - ON vector_generation_retirements ( - retired_by_worktree_id, snapshot_id, generation, retirement_id - ) WHERE retired_by_worktree_id IS NOT NULL`; - -const CORE_SCHEMA_TRIGGER_GUARD_SQL = `SELECT CASE - WHEN NOT EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'index' - AND name = 'vector_pointer_generation_lookup' - AND tbl_name = 'vector_pointers' - LIMIT 1 - ) OR ( - SELECT COUNT(*) FROM ( - SELECT seqno, cid, name, "desc", coll, "key" - FROM pragma_index_xinfo('vector_pointer_generation_lookup') - LIMIT 3 - ) - ) <> 2 OR ( - SELECT COUNT(*) FROM ( - SELECT seqno, cid, name, "desc", coll, "key" - FROM pragma_index_xinfo('vector_pointer_generation_lookup') - LIMIT 3 - ) WHERE ( - seqno = 0 AND name = 'generation' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 - ) OR ( - seqno = 1 AND cid = -1 AND name IS NULL AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 - ) - ) <> 2 OR NOT EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'table' - AND name = 'vectors' - AND tbl_name = 'vectors' - LIMIT 1 - ) OR ( - SELECT COUNT(*) FROM ( - SELECT seqno, cid, name, "desc", coll, "key" - FROM pragma_index_xinfo('sqlite_autoindex_vectors_1') - LIMIT 5 - ) - ) <> 4 OR ( - SELECT COUNT(*) FROM ( - SELECT seqno, cid, name, "desc", coll, "key" - FROM pragma_index_xinfo('sqlite_autoindex_vectors_1') - LIMIT 5 - ) WHERE ( - seqno = 0 AND name = 'generation' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 - ) OR ( - seqno = 1 AND name = 'symbol_id' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 - ) OR ( - seqno = 2 AND name = 'fingerprint' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 - ) OR ( - seqno = 3 AND name = 'vector' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 - ) - ) <> 4 - THEN RAISE(ABORT, 'code graph vector retirement authority is incompatible') -END;`; - -const RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL = `${CORE_SCHEMA_TRIGGER_GUARD_SQL} -SELECT CASE - WHEN NOT EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'table' - AND name = 'vector_retirement_state' - AND tbl_name = 'vector_retirement_state' - AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL))} - LIMIT 1 - ) OR NOT EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'table' - AND name = 'vector_generation_retirements' - AND tbl_name = 'vector_generation_retirements' - AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL))} - LIMIT 1 - ) OR NOT EXISTS ( - SELECT 1 FROM sqlite_master - WHERE type = 'index' - AND name = 'vector_generation_retirement_association' - AND tbl_name = 'vector_generation_retirements' - AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL))} - LIMIT 1 - ) OR ( - SELECT COUNT(*) FROM ( - SELECT name, typeof(seq) AS seq_type, seq - FROM sqlite_sequence - WHERE name = 'vector_generation_retirements' COLLATE NOCASE - LIMIT 2 - ) - ) <> 1 OR NOT EXISTS ( - SELECT 1 FROM sqlite_sequence - WHERE name = 'vector_generation_retirements' - AND typeof(seq) = 'integer' - AND seq BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND seq >= COALESCE(( - SELECT retirement_id - FROM vector_generation_retirements - ORDER BY retirement_id DESC - LIMIT 1 - ), 0) - LIMIT 1 - ) OR ( - SELECT COUNT(*) FROM (SELECT singleton FROM vector_retirement_state LIMIT 2) - ) <> 1 OR NOT EXISTS ( - SELECT 1 FROM vector_retirement_state - WHERE singleton = 1 - AND typeof(generation_revision) = 'integer' - AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND ( - ( - admission_cursor IS NULL - AND admission_scan_revision IS NULL - ) OR ( - typeof(admission_cursor) = 'text' - AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(admission_cursor, char(0)) = 0 - AND typeof(admission_scan_revision) = 'integer' - AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND admission_scan_revision <= generation_revision - ) - ) - AND ( - clean_generation_revision IS NULL OR ( - typeof(clean_generation_revision) = 'integer' - AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - AND clean_generation_revision <= generation_revision - ) - ) - AND ( - admission_scan_revision IS NULL - OR clean_generation_revision IS NULL - OR clean_generation_revision <= admission_scan_revision - ) - AND ( - clean_generation_revision IS NULL - OR clean_generation_revision < generation_revision - OR (admission_cursor IS NULL AND admission_scan_revision IS NULL) - ) - AND ( - ( - pointer_delete_worktree_id IS NULL - AND pointer_delete_generation IS NULL - AND pointer_delete_snapshot_id IS NULL - ) OR ( - typeof(pointer_delete_worktree_id) = 'text' - AND length(CAST(pointer_delete_worktree_id AS BLOB)) = 64 - AND pointer_delete_worktree_id NOT GLOB '*[^0-9a-f]*' - AND typeof(pointer_delete_generation) = 'text' - AND length(CAST(pointer_delete_generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(pointer_delete_generation, char(0)) = 0 - AND typeof(pointer_delete_snapshot_id) = 'text' - AND length(CAST(pointer_delete_snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} - AND instr(pointer_delete_snapshot_id, char(0)) = 0 - ) - ) - LIMIT 1 - ) - THEN RAISE(ABORT, 'code graph vector retirement marker authority is incompatible') -END;`; - -const VECTOR_RETIREMENT_MARKER_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_insert_guard - BEFORE INSERT ON vector_generation_retirements - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT CASE - WHEN NEW.retirement_id <> -1 - OR NEW.page_revision <> 0 - OR NEW.delete_authorized <> 0 - OR NOT EXISTS ( - SELECT 1 FROM vector_generations AS generation - WHERE generation.generation = NEW.generation - AND generation.snapshot_id = NEW.snapshot_id - AND ${vectorGenerationManifestPredicate('generation')} - LIMIT 1 - ) - OR EXISTS ( - SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup - WHERE generation = NEW.generation LIMIT 1 - ) - THEN RAISE(ABORT, 'code graph vector retirement marker is invalid') - END; - END`; - -const VECTOR_RETIREMENT_MARKER_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_update_guard - BEFORE UPDATE ON vector_generation_retirements - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT CASE - WHEN NEW.retirement_id <> OLD.retirement_id - OR NEW.generation <> OLD.generation - OR NEW.snapshot_id <> OLD.snapshot_id - OR NEW.retired_by_worktree_id IS NOT OLD.retired_by_worktree_id - OR NOT EXISTS ( - SELECT 1 FROM vector_generations AS generation - WHERE generation.generation = OLD.generation - AND generation.snapshot_id = OLD.snapshot_id - AND ${vectorGenerationManifestPredicate('generation')} - LIMIT 1 - ) - OR EXISTS ( - SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup - WHERE generation = OLD.generation LIMIT 1 - ) - OR NOT ( - ( - OLD.delete_authorized = 0 - AND NEW.delete_authorized = 0 - AND OLD.page_revision < ${MAXIMUM_SAFE_INTEGER_SQL} - AND NEW.page_revision = OLD.page_revision + 1 - ) OR ( - OLD.delete_authorized = 0 - AND NEW.delete_authorized = 1 - AND NEW.page_revision = OLD.page_revision - AND NOT EXISTS ( - SELECT 1 FROM vectors INDEXED BY sqlite_autoindex_vectors_1 - WHERE generation = OLD.generation LIMIT 1 - ) - ) - ) - THEN RAISE(ABORT, 'code graph vector retirement marker update is invalid') - END; - END`; - -const VECTOR_RETIREMENT_MARKER_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_delete_guard - BEFORE DELETE ON vector_generation_retirements - WHEN EXISTS ( - SELECT 1 FROM vector_generations WHERE generation = OLD.generation LIMIT 1 - ) - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT RAISE(ABORT, 'code graph vector retirement marker is still authoritative'); - END`; - -const POINTER_MANIFEST_TRIGGER_GUARD_SQL = `SELECT CASE - WHEN typeof(NEW.worktree_id) <> 'text' - OR length(CAST(NEW.worktree_id AS BLOB)) <> 64 - OR NEW.worktree_id GLOB '*[^0-9a-f]*' - OR typeof(NEW.generation) <> 'text' - OR length(CAST(NEW.generation AS BLOB)) NOT BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - OR instr(NEW.generation, char(0)) <> 0 - OR NOT EXISTS ( - SELECT 1 FROM vector_generations AS generation - WHERE generation.generation = NEW.generation - AND ${vectorGenerationManifestPredicate('generation')} - LIMIT 1 - ) - THEN RAISE(ABORT, 'code graph vector pointer manifest is invalid') -END;`; - -const OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL = POINTER_MANIFEST_TRIGGER_GUARD_SQL.replaceAll('NEW.', 'OLD.'); - -const VECTOR_RETIREMENT_POINTER_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_insert_guard - BEFORE INSERT ON vector_pointers - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - ${POINTER_MANIFEST_TRIGGER_GUARD_SQL} - SELECT CASE WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements - INDEXED BY sqlite_autoindex_vector_generation_retirements_1 - WHERE generation = NEW.generation LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; - END`; - -const VECTOR_RETIREMENT_POINTER_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_update_guard - BEFORE UPDATE ON vector_pointers - WHEN NEW.worktree_id <> OLD.worktree_id OR NEW.generation <> OLD.generation - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - ${OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL} - ${POINTER_MANIFEST_TRIGGER_GUARD_SQL} - SELECT CASE WHEN NEW.worktree_id <> OLD.worktree_id - THEN RAISE(ABORT, 'code graph vector pointer identity is immutable') END; - SELECT CASE WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements - INDEXED BY sqlite_autoindex_vector_generation_retirements_1 - WHERE generation = NEW.generation LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; - END`; - -const VECTOR_RETIREMENT_POINTER_DELETE_GUARD_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_delete_guard - BEFORE DELETE ON vector_pointers - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - ${OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL} - SELECT CASE WHEN NOT EXISTS ( - SELECT 1 - FROM vector_retirement_state AS authority - JOIN vector_generations AS generation - ON generation.generation = OLD.generation - AND generation.snapshot_id = authority.pointer_delete_snapshot_id - WHERE authority.singleton = 1 - AND authority.pointer_delete_worktree_id = OLD.worktree_id - AND authority.pointer_delete_generation = OLD.generation - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector pointer deletion is unauthorized') END; - END`; - -const VECTOR_RETIREMENT_POINTER_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_delete_mark - AFTER DELETE ON vector_pointers - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - INSERT INTO vector_generation_retirements ( - generation, snapshot_id, retired_by_worktree_id - ) - SELECT generation, snapshot_id, OLD.worktree_id - FROM vector_generations - WHERE generation = OLD.generation - AND NOT EXISTS ( - SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup - WHERE generation = OLD.generation LIMIT 1 - ); - UPDATE vector_retirement_state - SET pointer_delete_worktree_id = NULL, - pointer_delete_generation = NULL, - pointer_delete_snapshot_id = NULL - WHERE singleton = 1 - AND pointer_delete_worktree_id = OLD.worktree_id - AND pointer_delete_generation = OLD.generation; - SELECT CASE WHEN EXISTS ( - SELECT 1 FROM vector_retirement_state - WHERE singleton = 1 AND pointer_delete_worktree_id IS NOT NULL - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector pointer deletion authority was not consumed') END; - END`; - -const VECTOR_RETIREMENT_POINTER_CHANGED_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_update_mark - AFTER UPDATE OF generation ON vector_pointers - WHEN NEW.generation <> OLD.generation AND NOT EXISTS ( - SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup - WHERE generation = OLD.generation LIMIT 1 - ) - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - INSERT INTO vector_generation_retirements ( - generation, snapshot_id, retired_by_worktree_id - ) - SELECT generation, snapshot_id, OLD.worktree_id - FROM vector_generations - WHERE generation = OLD.generation; - END`; - -const VECTOR_RETIREMENT_VECTOR_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_vector_insert_guard - BEFORE INSERT ON vectors - WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements - INDEXED BY sqlite_autoindex_vector_generation_retirements_1 - WHERE generation = NEW.generation LIMIT 1 - ) - BEGIN - SELECT RAISE(ABORT, 'code graph vector generation is retiring'); - END`; - -const VECTOR_RETIREMENT_VECTOR_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_vector_update_guard - BEFORE UPDATE ON vectors - WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements - INDEXED BY sqlite_autoindex_vector_generation_retirements_1 - WHERE generation = OLD.generation OR generation = NEW.generation - LIMIT 1 - ) - BEGIN - SELECT RAISE(ABORT, 'code graph vector generation is retiring'); - END`; - -const VECTOR_RETIREMENT_GENERATION_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_insert_guard - BEFORE INSERT ON vector_generations - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT CASE WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements WHERE generation = NEW.generation LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; - SELECT CASE WHEN NOT EXISTS ( - SELECT 1 FROM vector_retirement_state - WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; - UPDATE vector_retirement_state - SET generation_revision = generation_revision + 1 - WHERE singleton = 1; - END`; - -const VECTOR_RETIREMENT_GENERATION_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_update_guard - BEFORE UPDATE ON vector_generations - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT CASE WHEN EXISTS ( - SELECT 1 FROM vector_generation_retirements - WHERE generation = OLD.generation OR generation = NEW.generation - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; - SELECT CASE WHEN NOT EXISTS ( - SELECT 1 FROM vector_retirement_state - WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; - UPDATE vector_retirement_state - SET generation_revision = generation_revision + 1 - WHERE singleton = 1; - END`; - -const VECTOR_RETIREMENT_GENERATION_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_delete_guard - BEFORE DELETE ON vector_generations - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - SELECT CASE - WHEN NOT EXISTS ( - SELECT 1 FROM vector_generation_retirements - WHERE generation = OLD.generation AND delete_authorized = 1 - LIMIT 1 - ) OR EXISTS ( - SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup - WHERE generation = OLD.generation LIMIT 1 - ) OR EXISTS ( - SELECT 1 FROM vectors INDEXED BY sqlite_autoindex_vectors_1 - WHERE generation = OLD.generation LIMIT 1 - ) - THEN RAISE(ABORT, 'code graph vector generation deletion is unauthorized') - END; - SELECT CASE WHEN NOT EXISTS ( - SELECT 1 FROM vector_retirement_state - WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} - LIMIT 1 - ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; - UPDATE vector_retirement_state - SET generation_revision = generation_revision + 1 - WHERE singleton = 1; - END`; - -const VECTOR_RETIREMENT_GENERATION_DELETED_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_deleted_clear - AFTER DELETE ON vector_generations - BEGIN - ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} - DELETE FROM vector_generation_retirements - WHERE generation = OLD.generation AND delete_authorized = 1; - END`; - -export const CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS = [ - {name: 'vector_retirement_marker_insert_guard', sql: VECTOR_RETIREMENT_MARKER_INSERT_TRIGGER_SQL}, - {name: 'vector_retirement_marker_update_guard', sql: VECTOR_RETIREMENT_MARKER_UPDATE_TRIGGER_SQL}, - {name: 'vector_retirement_marker_delete_guard', sql: VECTOR_RETIREMENT_MARKER_DELETE_TRIGGER_SQL}, - {name: 'vector_retirement_pointer_insert_guard', sql: VECTOR_RETIREMENT_POINTER_INSERT_TRIGGER_SQL}, - {name: 'vector_retirement_pointer_update_guard', sql: VECTOR_RETIREMENT_POINTER_UPDATE_TRIGGER_SQL}, - {name: 'vector_retirement_pointer_delete_guard', sql: VECTOR_RETIREMENT_POINTER_DELETE_GUARD_TRIGGER_SQL}, - {name: 'vector_retirement_pointer_delete_mark', sql: VECTOR_RETIREMENT_POINTER_DELETE_TRIGGER_SQL}, - {name: 'vector_retirement_pointer_update_mark', sql: VECTOR_RETIREMENT_POINTER_CHANGED_TRIGGER_SQL}, - {name: 'vector_retirement_vector_insert_guard', sql: VECTOR_RETIREMENT_VECTOR_INSERT_TRIGGER_SQL}, - {name: 'vector_retirement_vector_update_guard', sql: VECTOR_RETIREMENT_VECTOR_UPDATE_TRIGGER_SQL}, - {name: 'vector_retirement_generation_insert_guard', sql: VECTOR_RETIREMENT_GENERATION_INSERT_TRIGGER_SQL}, - {name: 'vector_retirement_generation_update_guard', sql: VECTOR_RETIREMENT_GENERATION_UPDATE_TRIGGER_SQL}, - {name: 'vector_retirement_generation_delete_guard', sql: VECTOR_RETIREMENT_GENERATION_DELETE_TRIGGER_SQL}, - {name: 'vector_retirement_generation_deleted_clear', sql: VECTOR_RETIREMENT_GENERATION_DELETED_TRIGGER_SQL}, -] as const; - -// sqlite_schema rows (tables, implicit/explicit indexes, triggers), the -// singleton state row, and sqlite_sequence authority published by r1. The -// fixed conservative count is versioned by the exact SQL byte constant below. -export const CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_ROWS = 24; -export const CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_BYTES = [ +import { + boundedRetirementLimit, + codeGraphVectorCoreSchemaCurrent, + codeGraphVectorCoreSchemaState, + codeGraphVectorRetirementSchemaState, + inspectCodeGraphVectorPageStorage, + inspectLegacyPointerIndexPlan, + inspectVectorPageStorageSql, + lastStatementChangeCount, + sameLegacyPointerIndexPlan, + sameVectorPageStorage, + selectVectorRetirementMarker, + useExistingVectorDatabase, + useReadOnlyVectorDatabase, + validBoundedText, + vectorRetirementPageAuthorityBytes, + type CodeGraphVectorPageStorage, + type CodeGraphVectorRetirementMarker, + type LegacyPointerIndexPlan, +} from './vector_retirement_inspection.js'; +import { + CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL, + CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_FIXED_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_ROWS, CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS, + CodeGraphVectorRetirementError, + MAXIMUM_SAFE_INTEGER_SQL, + VECTOR_CREATED_AT_BYTES, + VECTOR_FINGERPRINT_BYTES, + VECTOR_GENERATION_BYTES, + VECTOR_MODEL_ID_BYTES, + VECTOR_MODEL_SHA256_BYTES, + VECTOR_SNAPSHOT_BYTES, + VECTOR_SYMBOL_BYTES, + storedSchemaSql, +} from './vector_retirement_schema.js'; + +export { + codeGraphVectorRetirementLegacyPointerProbeStatement, + inspectCodeGraphVectorPageStorage, + selectCodeGraphVectorRetirementMarker, + type CodeGraphVectorPageStorage, + type CodeGraphVectorRetirementMarker, + type LegacyPointerIndexPlan, +} from './vector_retirement_inspection.js'; +export { + CODE_GRAPH_VECTORS_TABLE_SQL, + CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL, + CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL, + CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL, CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, - ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map(trigger => trigger.sql), -].reduce((total, sql) => total + new TextEncoder().encode(storedSchemaSql(sql)).byteLength, 256); - -export interface CodeGraphVectorRetirementMarker { - readonly deleteAuthorized: boolean; - readonly generation: string; - readonly pageRevision: number; - readonly retiredByWorktreeId?: string; - readonly retirementId: number; - readonly snapshotId: string; -} + CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_FIXED_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS, + CODE_GRAPH_VECTOR_REUSE_INDEX_SQL, +} from './vector_retirement_schema.js'; export type CodeGraphVectorRetirementPreparationResult = {readonly state: 'prepared' | 'ready'}; @@ -758,7 +186,9 @@ const observeCodeGraphVectorRetirementCapacity = Effect.fn('codeGraph.observeVec ); const pageStorage = Option.getOrUndefined(storage); if (pageStorage === undefined || !sameVectorPageStorage(input.storage, pageStorage)) { - return yield* Effect.fail(new Error('Code graph vector page storage changed before reservation.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector page storage changed before reservation.'), + ); } const durableDevice = Option.isSome(durableInfo) ? durableInfo.value.dev : undefined; const temporaryDevice = Option.isSome(temporaryInfo) ? temporaryInfo.value.dev : undefined; @@ -855,15 +285,21 @@ const observeVectorPointerRetirement = Effect.fn('codeGraph.observeVectorPointer typeof row.generation !== 'string' || typeof row.snapshot_id !== 'string' ) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority is invalid.'), + ); } if (row.snapshot_id !== input.expectedSnapshotId) return undefined; const generationManifest = yield* inspectBoundedVectorGenerationManifest(sql, row.generation); if (generationManifest.snapshotId !== input.expectedSnapshotId) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority changed.'), + ); } if ((yield* selectVectorRetirementMarker(sql, row.generation)) !== undefined) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement marker is already authoritative.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement marker is already authoritative.'), + ); } return {generationManifest, worktreeId: input.worktreeId} satisfies CodeGraphVectorPointerRetirementObservation; }); @@ -883,7 +319,9 @@ export const planCodeGraphVectorPointerRetirement = Effect.fn('codeGraph.planVec input: CodeGraphVectorPointerRetirementInput, ) { if (!/^[0-9a-f]{64}$/.test(input.worktreeId) || !validBoundedText(input.expectedSnapshotId, VECTOR_SNAPSHOT_BYTES)) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement target is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement target is invalid.'), + ); } return yield* useExistingVectorDatabase( databasePath, @@ -895,7 +333,9 @@ export const planCodeGraphVectorPointerRetirement = Effect.fn('codeGraph.planVec !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const observation = yield* observeVectorPointerRetirement(sql, input); if (observation === undefined) { @@ -938,12 +378,16 @@ export const commitCodeGraphVectorPointerRetirement = Effect.fn('codeGraph.commi (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' || !sameVectorPageStorage(plan.storage, yield* inspectVectorPageStorageSql(sql)) ) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority changed.'), + ); } const observed = yield* observeVectorPointerRetirement(sql, plan.input); if (observed === undefined) return 0; if (!sameVectorPointerRetirementObservation(plan.observation, observed)) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement plan changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement plan changed.'), + ); } yield* sql.unsafe( `UPDATE vector_retirement_state @@ -957,14 +401,18 @@ export const commitCodeGraphVectorPointerRetirement = Effect.fn('codeGraph.commi [plan.input.worktreeId, observed.generationManifest.generation, observed.generationManifest.snapshotId], ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority is busy.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority is busy.'), + ); } yield* sql.unsafe('DELETE FROM vector_pointers WHERE worktree_id = ? AND generation = ?', [ plan.input.worktreeId, observed.generationManifest.generation, ]); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement target changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement target changed.'), + ); } const authority = yield* sql.unsafe( `SELECT 1 FROM vector_retirement_state @@ -975,7 +423,9 @@ export const commitCodeGraphVectorPointerRetirement = Effect.fn('codeGraph.commi LIMIT 1`, ); if (authority.length !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority was retained.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority was retained.'), + ); } return 1; }), @@ -1011,7 +461,9 @@ export const deleteCodeGraphVectorPointerWithRetirement = Effect.fn('codeGraph.d !/^[0-9a-f]{64}$/.test(input.worktreeId) || !validBoundedText(input.expectedSnapshotId, VECTOR_SNAPSHOT_BYTES) ) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement target is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement target is invalid.'), + ); } return yield* useExistingVectorDatabase( databasePath, @@ -1023,7 +475,9 @@ export const deleteCodeGraphVectorPointerWithRetirement = Effect.fn('codeGraph.d !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } return yield* deleteCodeGraphVectorPointerWithRetirementSql(sql, input); }), @@ -1040,7 +494,9 @@ export const deleteCodeGraphVectorPointerWithRetirementSql = Effect.fn( !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const rows = yield* sql.unsafe<{ readonly generation: unknown; @@ -1068,7 +524,9 @@ export const deleteCodeGraphVectorPointerWithRetirementSql = Effect.fn( if (rows.length === 0) return 0; const row = rows[0]; if (rows.length !== 1 || typeof row?.generation !== 'string' || typeof row.snapshot_id !== 'string') { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority is invalid.'), + ); } if (row.snapshot_id !== input.expectedSnapshotId) return 0; yield* sql.unsafe( @@ -1083,14 +541,18 @@ export const deleteCodeGraphVectorPointerWithRetirementSql = Effect.fn( [input.worktreeId, row.generation, row.snapshot_id], ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority is busy.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority is busy.'), + ); } yield* sql.unsafe('DELETE FROM vector_pointers WHERE worktree_id = ? AND generation = ?', [ input.worktreeId, row.generation, ]); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement target changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement target changed.'), + ); } const authority = yield* sql.unsafe( `SELECT 1 FROM vector_retirement_state @@ -1101,43 +563,15 @@ export const deleteCodeGraphVectorPointerWithRetirementSql = Effect.fn( LIMIT 1`, ); if (authority.length !== 1) { - return yield* Effect.fail(new Error('Code graph vector pointer retirement authority was retained.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer retirement authority was retained.'), + ); } return 1; }), ); }); -export function codeGraphVectorRetirementLegacyPointerProbeStatement() { - return { - parameters: [CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS + 1] as const, - text: `SELECT - CASE - WHEN typeof(worktree_id) = 'text' - AND length(CAST(worktree_id AS BLOB)) = 64 - AND worktree_id NOT GLOB '*[^0-9a-f]*' - THEN worktree_id ELSE NULL - END AS worktree_id, - CASE - WHEN typeof(generation) = 'text' - AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(generation, char(0)) = 0 - THEN generation ELSE NULL - END AS generation, - length(CAST(worktree_id AS BLOB)) + length(CAST(generation AS BLOB)) AS identity_bytes - FROM vector_pointers - ORDER BY vector_pointers.worktree_id - LIMIT ?`, - }; -} - -/** @internal Frozen manifest for the released-v2 pointer-index bridge. */ -export interface LegacyPointerIndexPlan { - readonly finalFactBytes: number; - readonly rows: readonly {readonly generation: string; readonly worktreeId: string}[]; - readonly storage: CodeGraphVectorPageStorage; -} - export type CodeGraphVectorRetirementPreparationPlan = | {readonly state: 'ready'} | { @@ -1159,17 +593,23 @@ export const planCodeGraphVectorRetirementPreparation = Effect.fn('codeGraph.pla yield* sql.unsafe('PRAGMA busy_timeout = 0'); const versions = yield* sql.unsafe<{readonly user_version: unknown}>('PRAGMA user_version'); if (versions.length !== 1 || versions[0]?.user_version !== 2) { - return yield* Effect.fail(new Error('Code graph vector database version is unsupported.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector database version is unsupported.'), + ); } const coreState = yield* codeGraphVectorCoreSchemaState(sql); if (coreState === 'incompatible') { - return yield* Effect.fail(new Error('Code graph vector database authority is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector database authority is incompatible.'), + ); } if (coreState === 'ready') { const retirementState = yield* codeGraphVectorRetirementSchemaState(sql); if (retirementState === 'ready') return {state: 'ready'} as const; if (retirementState === 'incompatible') { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } return { coreState, @@ -1179,7 +619,9 @@ export const planCodeGraphVectorRetirementPreparation = Effect.fn('codeGraph.pla } as const; } if ((yield* codeGraphVectorRetirementSchemaState(sql)) !== 'absent') { - return yield* Effect.fail(new Error('Code graph vector retirement authority is incomplete.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement authority is incomplete.'), + ); } const legacy = yield* inspectLegacyPointerIndexPlan(sql); return { @@ -1226,22 +668,30 @@ export const commitCodeGraphVectorRetirementPreparation = Effect.fn('codeGraph.c coreState !== plan.coreState || !sameVectorPageStorage(plan.storage, yield* inspectVectorPageStorageSql(sql)) ) { - return yield* Effect.fail(new Error('Code graph vector database authority changed during setup.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector database authority changed during setup.'), + ); } if ((yield* codeGraphVectorRetirementSchemaState(sql)) !== plan.retirementState) { - return yield* Effect.fail(new Error('Code graph vector retirement authority changed during setup.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement authority changed during setup.'), + ); } if (coreState === 'missing-pointer-index') { const revalidated = yield* inspectLegacyPointerIndexPlan(sql); if (plan.legacy === undefined || !sameLegacyPointerIndexPlan(plan.legacy, revalidated)) { - return yield* Effect.fail(new Error('Code graph vector pointer index plan changed during setup.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer index plan changed during setup.'), + ); } yield* sql.unsafe('PRAGMA temp_store = MEMORY'); yield* sql.unsafe(CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL); } const result = yield* publishCodeGraphVectorRetirementSchema(sql); if (!(yield* codeGraphVectorCoreSchemaCurrent(sql))) { - return yield* Effect.fail(new Error('Code graph vector database authority changed during setup.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector database authority changed during setup.'), + ); } return result; }), @@ -1267,12 +717,16 @@ export const prepareCodeGraphVectorRetirement = Effect.fn('codeGraph.prepareVect export const initializeCodeGraphVectorRetirementSchema = Effect.fn('codeGraph.initializeVectorRetirementSchema')( function* (sql: SqlClient.SqlClient) { if (!(yield* codeGraphVectorCoreSchemaCurrent(sql))) { - return yield* Effect.fail(new Error('Code graph vector database authority is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector database authority is incompatible.'), + ); } const state = yield* codeGraphVectorRetirementSchemaState(sql); if (state === 'ready') return {state: 'ready'} as const; if (state === 'incompatible') { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } return yield* sql.withTransaction(publishCodeGraphVectorRetirementSchema(sql)); }, @@ -1285,7 +739,9 @@ export const requireCodeGraphVectorRetirementSchema = Effect.fn('codeGraph.requi !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema requires explicit preparation.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema requires explicit preparation.'), + ); } }); @@ -1299,7 +755,9 @@ const publishCodeGraphVectorRetirementSchema = Effect.fn('codeGraph.publishVecto yield* sql.unsafe(CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL); for (const trigger of CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS) yield* sql.unsafe(trigger.sql); if ((yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready') { - return yield* Effect.fail(new Error('Code graph vector retirement schema changed during setup.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema changed during setup.'), + ); } return {state: 'prepared'} as const; }); @@ -1354,13 +812,6 @@ interface CodeGraphVectorGenerationManifest { readonly templateVersion: number; } -export interface CodeGraphVectorPageStorage { - readonly freelistBytes: number; - readonly journalMode: 'delete' | 'wal'; - readonly pageSize: number; - readonly walAutoCheckpointPages: number; -} - const inspectBoundedVectorRetirementPage = Effect.fn('codeGraph.inspectBoundedVectorRetirementPage')(function* ( sql: SqlClient.SqlClient, generation: string, @@ -1384,7 +835,9 @@ const inspectBoundedVectorRetirementPage = Effect.fn('codeGraph.inspectBoundedVe !Number.isSafeInteger(manifest.fingerprint_bytes) || !Number.isSafeInteger(manifest.vector_bytes) ) { - return yield* Effect.fail(new Error('Code graph vector retirement manifest is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement manifest is invalid.'), + ); } const rowBytes = Number(manifest.symbol_bytes) + @@ -1393,7 +846,9 @@ const inspectBoundedVectorRetirementPage = Effect.fn('codeGraph.inspectBoundedVe generationBytes + 64; if (!Number.isSafeInteger(rowBytes) || rowBytes <= 0) { - return yield* Effect.fail(new Error('Code graph vector retirement manifest is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement manifest is invalid.'), + ); } if (finalFactBytes + rowBytes > CODE_GRAPH_VECTOR_RETIREMENT_PAGE_BYTES) break; finalFactBytes += rowBytes; @@ -1401,7 +856,9 @@ const inspectBoundedVectorRetirementPage = Effect.fn('codeGraph.inspectBoundedVe rowCount += 1; } if (manifests.length > 0 && lastSymbolId === undefined) { - return yield* Effect.fail(new Error('Code graph vector retirement page exceeds its byte bound.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement page exceeds its byte bound.'), + ); } return {finalFactBytes, lastSymbolId, rowCount} satisfies BoundedVectorRetirementPage; }); @@ -1472,7 +929,7 @@ const inspectBoundedVectorGenerationManifest = Effect.fn('codeGraph.inspectBound (row.bounded_state !== 'building' && row.bounded_state !== 'ready') || typeof row.bounded_created_at !== 'string' ) { - return yield* Effect.fail(new Error('Code graph vector generation manifest is invalid.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector generation manifest is invalid.')); } const strings = [ row.bounded_generation, @@ -1484,7 +941,7 @@ const inspectBoundedVectorGenerationManifest = Effect.fn('codeGraph.inspectBound ]; const finalFactBytes = strings.reduce((total, value) => total + new TextEncoder().encode(value).byteLength, 128); if (!Number.isSafeInteger(finalFactBytes)) { - return yield* Effect.fail(new Error('Code graph vector generation manifest is invalid.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector generation manifest is invalid.')); } return { count: Number(row.bounded_count), @@ -1528,7 +985,7 @@ export const planCodeGraphVectorRetirementPage = Effect.fn('codeGraph.planVector !Number.isSafeInteger(expectedRetirementId) || Number(expectedRetirementId) <= 0 ) { - return yield* Effect.fail(new Error('Code graph vector retirement candidate is invalid.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector retirement candidate is invalid.')); } const requestedLimit = input.requestedLimit ?? CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS; const limit = boundedRetirementLimit(requestedLimit); @@ -1542,7 +999,9 @@ export const planCodeGraphVectorRetirementPage = Effect.fn('codeGraph.planVector !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const marker = yield* selectVectorRetirementMarker(sql, input.generation); if (marker === undefined || marker.retirementId !== expectedRetirementId) { @@ -1552,7 +1011,9 @@ export const planCodeGraphVectorRetirementPage = Effect.fn('codeGraph.planVector } satisfies CodeGraphVectorRetirementPagePlan; } if (marker.deleteAuthorized) { - return yield* Effect.fail(new Error('Code graph vector retirement authorization is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement authorization is invalid.'), + ); } const pointers = yield* sql.unsafe( `SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup @@ -1565,11 +1026,15 @@ export const planCodeGraphVectorRetirementPage = Effect.fn('codeGraph.planVector state: 'retired-generation', }); if (lifecycle.disposition !== 'reclaim') { - return yield* Effect.fail(new Error('Code graph vector retirement still has a live pointer.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement still has a live pointer.'), + ); } const generationManifest = yield* inspectBoundedVectorGenerationManifest(sql, marker.generation); if (generationManifest.snapshotId !== marker.snapshotId) { - return yield* Effect.fail(new Error('Code graph vector generation authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector generation authority changed.'), + ); } const page = yield* inspectBoundedVectorRetirementPage(sql, marker.generation, limit); return { @@ -1608,10 +1073,12 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } if (!sameVectorPageStorage(plan.storage, yield* inspectVectorPageStorageSql(sql))) { - return yield* Effect.fail(new Error('Code graph vector page storage changed.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector page storage changed.')); } const marker = yield* selectVectorRetirementMarker(sql, plan.generation); if ( @@ -1624,7 +1091,9 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe return {remaining: false, rowsDeleted: 0, state: 'stale'} as const; } if (marker.deleteAuthorized) { - return yield* Effect.fail(new Error('Code graph vector retirement authorization is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement authorization is invalid.'), + ); } const pointers = yield* sql.unsafe( `SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup @@ -1637,11 +1106,15 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe state: 'retired-generation', }); if (lifecycle.disposition !== 'reclaim') { - return yield* Effect.fail(new Error('Code graph vector retirement still has a live pointer.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement still has a live pointer.'), + ); } const generationManifest = yield* inspectBoundedVectorGenerationManifest(sql, marker.generation); if (!sameVectorGenerationManifest(plan.generationManifest, generationManifest)) { - return yield* Effect.fail(new Error('Code graph vector generation manifest changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector generation manifest changed.'), + ); } const page = yield* inspectBoundedVectorRetirementPage(sql, marker.generation, plan.requestedLimit); if ( @@ -1650,7 +1123,7 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe page.rowCount !== plan.selectedRowCount || page.lastSymbolId !== plan.lastSymbolId ) { - return yield* Effect.fail(new Error('Code graph vector retirement page changed.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector retirement page changed.')); } let rowsDeleted = 0; if (page.lastSymbolId !== undefined) { @@ -1661,7 +1134,9 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe ); rowsDeleted = yield* lastStatementChangeCount(sql); if (rowsDeleted !== page.rowCount) { - return yield* Effect.fail(new Error('Code graph vector retirement page changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement page changed.'), + ); } } const remaining = yield* sql.unsafe( @@ -1678,7 +1153,9 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe [marker.generation, marker.retirementId, marker.pageRevision], ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement marker changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker changed.'), + ); } return { marker: {...marker, pageRevision: marker.pageRevision + 1}, @@ -1695,11 +1172,15 @@ export const commitCodeGraphVectorRetirementPage = Effect.fn('codeGraph.commitVe [marker.generation, marker.retirementId, marker.pageRevision], ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement authorization changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement authorization changed.'), + ); } yield* sql.unsafe('DELETE FROM vector_generations WHERE generation = ?', [marker.generation]); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement generation changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement generation changed.'), + ); } return {remaining: false, rowsDeleted, state: 'complete'} as const; }), @@ -1811,7 +1292,9 @@ const observeVectorRetirementAdmission = Effect.fn('codeGraph.observeVectorRetir Number(rawCleanGenerationRevision) === Number(rawGenerationRevision) && (rawCursor !== null || rawAdmissionScanRevision !== null)) ) { - return yield* Effect.fail(new Error('Code graph vector retirement admission state is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission state is invalid.'), + ); } const cursor = typeof rawCursor === 'string' ? rawCursor : undefined; const generationRevision = Number(rawGenerationRevision); @@ -1848,7 +1331,9 @@ const observeVectorRetirementAdmission = Effect.fn('codeGraph.observeVectorRetir } const generation = rows[0]?.generation; if (typeof generation !== 'string') { - return yield* Effect.fail(new Error('Code graph vector retirement admission row is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission row is invalid.'), + ); } const candidate = yield* inspectBoundedVectorGenerationManifest(sql, generation); const marker = yield* selectVectorRetirementMarker(sql, generation); @@ -1858,10 +1343,14 @@ const observeVectorRetirementAdmission = Effect.fn('codeGraph.observeVectorRetir [generation], ); if (marker !== undefined && pointers.length !== 0) { - return yield* Effect.fail(new Error('Code graph vector retirement admission authority is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission authority is invalid.'), + ); } if (marker !== undefined && (marker.snapshotId !== candidate.snapshotId || marker.deleteAuthorized)) { - return yield* Effect.fail(new Error('Code graph vector retirement admission marker is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission marker is invalid.'), + ); } return { ...(admissionScanRevision === undefined ? {} : {admissionScanRevision}), @@ -1934,7 +1423,9 @@ const applyVectorRetirementAdmissionObservation = Effect.fn('codeGraph.applyVect exactStateParameters, ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement admission revision changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission revision changed.'), + ); } return {state: 'restarted'} as const; } @@ -1955,7 +1446,9 @@ const applyVectorRetirementAdmissionObservation = Effect.fn('codeGraph.applyVect exactStateParameters, ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement admission cursor changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission cursor changed.'), + ); } return {state: 'wrapped'} as const; } @@ -1969,7 +1462,9 @@ const applyVectorRetirementAdmissionObservation = Effect.fn('codeGraph.applyVect ); marker = yield* selectVectorRetirementMarker(sql, observed.candidate.generation); if (marker === undefined) { - return yield* Effect.fail(new Error('Code graph vector retirement marker was not published.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker was not published.'), + ); } } yield* sql.unsafe( @@ -1987,7 +1482,9 @@ const applyVectorRetirementAdmissionObservation = Effect.fn('codeGraph.applyVect [observed.candidate.generation, ...exactStateParameters], ); if ((yield* lastStatementChangeCount(sql)) !== 1) { - return yield* Effect.fail(new Error('Code graph vector retirement admission cursor changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission cursor changed.'), + ); } return marker === undefined ? ({generation: observed.candidate.generation, state: 'advanced'} as const) @@ -2008,7 +1505,9 @@ export const planCodeGraphVectorRetirementAdmission = Effect.fn('codeGraph.planV !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const observation: CodeGraphVectorRetirementAdmissionObservation = yield* observeVectorRetirementAdmission(sql); if ( @@ -2057,12 +1556,16 @@ export const commitCodeGraphVectorRetirementAdmission = Effect.fn('codeGraph.com (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' || !sameVectorPageStorage(plan.storage, yield* inspectVectorPageStorageSql(sql)) ) { - return yield* Effect.fail(new Error('Code graph vector retirement admission authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission authority changed.'), + ); } const observed: CodeGraphVectorRetirementAdmissionObservation = yield* observeVectorRetirementAdmission(sql); if (!sameVectorRetirementAdmissionObservation(plan.observation, observed)) { - return yield* Effect.fail(new Error('Code graph vector retirement admission plan changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement admission plan changed.'), + ); } return yield* applyVectorRetirementAdmissionObservation(sql, observed); }), @@ -2132,7 +1635,7 @@ export const inspectCodeGraphVectorSnapshotUsage = Effect.fn('codeGraph.inspectV snapshotId: string, ) { if (!validBoundedText(snapshotId, VECTOR_SNAPSHOT_BYTES)) { - return yield* Effect.fail(new Error('Code graph vector snapshot identity is invalid.')); + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector snapshot identity is invalid.')); } return yield* useReadOnlyVectorDatabase( databasePath, @@ -2143,7 +1646,9 @@ export const inspectCodeGraphVectorSnapshotUsage = Effect.fn('codeGraph.inspectV !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const boundedLimit = CODE_GRAPH_VECTOR_SNAPSHOT_USAGE_LIMIT + 1; const generationRows = yield* sql.unsafe<{ @@ -2179,7 +1684,9 @@ export const inspectCodeGraphVectorSnapshotUsage = Effect.fn('codeGraph.inspectV generationRows.length > CODE_GRAPH_VECTOR_SNAPSHOT_USAGE_LIMIT || pointerRows.length > CODE_GRAPH_VECTOR_SNAPSHOT_USAGE_LIMIT ) { - return yield* Effect.fail(new Error('Code graph vector snapshot evidence exceeded its bound.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector snapshot evidence exceeded its bound.'), + ); } const generations = generationRows.map(row => { if ( @@ -2224,7 +1731,9 @@ export const inspectCodeGraphVectorSnapshotUsage = Effect.fn('codeGraph.inspectV return [row.generation, row.worktree_id] as const; }); if (generations.some(row => row === undefined) || pointers.some(row => row === undefined)) { - return yield* Effect.fail(new Error('Code graph vector snapshot evidence is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector snapshot evidence is invalid.'), + ); } return { activePointerCount: pointers.length, @@ -2249,7 +1758,9 @@ export const inspectCodeGraphVectorRetirementWork = Effect.fn('codeGraph.inspect !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const revisionStatement = codeGraphVectorRetirementCleanRevisionProbeStatement(); const revisionRows = yield* sql.unsafe<{readonly clean: unknown}>( @@ -2257,7 +1768,9 @@ export const inspectCodeGraphVectorRetirementWork = Effect.fn('codeGraph.inspect revisionStatement.parameters, ); if (revisionRows.length !== 1 || (revisionRows[0]?.clean !== 0 && revisionRows[0]?.clean !== 1)) { - return yield* Effect.fail(new Error('Code graph vector retirement clean revision is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement clean revision is invalid.'), + ); } if (revisionRows[0].clean === 0) return {state: 'admission'} as const; @@ -2270,11 +1783,15 @@ export const inspectCodeGraphVectorRetirementWork = Effect.fn('codeGraph.inspect const generation = markerRows[0]?.generation; const retirementId = markerRows[0]?.retirement_id; if (typeof generation !== 'string' || !Number.isSafeInteger(retirementId) || Number(retirementId) <= 0) { - return yield* Effect.fail(new Error('Code graph vector retirement marker probe is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker probe is invalid.'), + ); } const marker = yield* selectVectorRetirementMarker(sql, generation); if (marker === undefined || marker.retirementId !== retirementId) { - return yield* Effect.fail(new Error('Code graph vector retirement marker authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker authority changed.'), + ); } return {generation, state: 'marker'} as const; }), @@ -2332,7 +1849,9 @@ export const selectCodeGraphVectorRetirementMarkerCandidate = Effect.fn( (input.retiredByWorktreeId !== undefined && !/^[0-9a-f]{64}$/.test(input.retiredByWorktreeId)) || (input.snapshotId !== undefined && !validBoundedText(input.snapshotId, VECTOR_SNAPSHOT_BYTES)) ) { - return yield* Effect.fail(new Error('Code graph vector retirement marker selector is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker selector is invalid.'), + ); } return yield* useReadOnlyVectorDatabase( databasePath, @@ -2342,7 +1861,9 @@ export const selectCodeGraphVectorRetirementMarkerCandidate = Effect.fn( !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const statement = codeGraphVectorRetirementMarkerPageStatement(input); const rows = yield* sql.unsafe<{readonly generation: unknown; readonly retirement_id: unknown}>( @@ -2353,7 +1874,9 @@ export const selectCodeGraphVectorRetirementMarkerCandidate = Effect.fn( const generation = rows[0]?.generation; const retirementId = rows[0]?.retirement_id; if (typeof generation !== 'string' || !Number.isSafeInteger(retirementId) || Number(retirementId) <= 0) { - return yield* Effect.fail(new Error('Code graph vector retirement marker selector is invalid.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker selector is invalid.'), + ); } const marker = yield* selectVectorRetirementMarker(sql, generation); if ( @@ -2362,7 +1885,9 @@ export const selectCodeGraphVectorRetirementMarkerCandidate = Effect.fn( (input.retiredByWorktreeId !== undefined && (marker.retiredByWorktreeId !== input.retiredByWorktreeId || marker.snapshotId !== input.snapshotId)) ) { - return yield* Effect.fail(new Error('Code graph vector retirement marker authority changed.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement marker authority changed.'), + ); } return marker; }), @@ -2379,884 +1904,12 @@ export const admitOneCodeGraphVectorRetirement = Effect.fn('codeGraph.admitVecto !(yield* codeGraphVectorCoreSchemaCurrent(sql)) || (yield* codeGraphVectorRetirementSchemaState(sql)) !== 'ready' ) { - return yield* Effect.fail(new Error('Code graph vector retirement schema is incompatible.')); + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement schema is incompatible.'), + ); } const observed = yield* observeVectorRetirementAdmission(sql); return yield* applyVectorRetirementAdmissionObservation(sql, observed); }), ); }); - -interface ExpectedVectorIndexColumn { - readonly cid: number; - readonly coll: 'BINARY'; - readonly desc: 0; - readonly key: 0 | 1; - readonly name: string | null; -} - -interface ExpectedVectorIndex { - readonly columns: readonly ExpectedVectorIndexColumn[]; - readonly name: string; - readonly origin: 'c' | 'pk' | 'u'; - readonly partial: 0 | 1; - readonly unique: 0 | 1; -} - -interface ExpectedVectorForeignKey { - readonly from: string; - readonly id: number; - readonly match: 'NONE'; - readonly onDelete: 'CASCADE' | 'NO ACTION'; - readonly onUpdate: 'NO ACTION'; - readonly seq: number; - readonly table: string; - readonly to: string; -} - -const rowIdPayload = {cid: -1, coll: 'BINARY', desc: 0, key: 0, name: null} as const; - -const VECTOR_GENERATION_INDEXES = [ - { - columns: [{cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], - name: 'sqlite_autoindex_vector_generations_1', - origin: 'pk', - partial: 0, - unique: 1, - }, -] as const satisfies readonly ExpectedVectorIndex[]; - -const VECTOR_POINTER_PRIMARY_INDEX = { - columns: [{cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'worktree_id'}, rowIdPayload], - name: 'sqlite_autoindex_vector_pointers_1', - origin: 'pk', - partial: 0, - unique: 1, -} as const satisfies ExpectedVectorIndex; - -const VECTOR_POINTER_INDEXES_WITHOUT_GENERATION = [VECTOR_POINTER_PRIMARY_INDEX] as const; -const VECTOR_POINTER_INDEXES = [ - VECTOR_POINTER_PRIMARY_INDEX, - { - columns: [{cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], - name: 'vector_pointer_generation_lookup', - origin: 'c', - partial: 0, - unique: 0, - }, -] as const satisfies readonly ExpectedVectorIndex[]; - -const VECTOR_ROW_INDEXES = [ - { - columns: [ - {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, - {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'symbol_id'}, - {cid: 2, coll: 'BINARY', desc: 0, key: 0, name: 'fingerprint'}, - {cid: 3, coll: 'BINARY', desc: 0, key: 0, name: 'vector'}, - ], - name: 'sqlite_autoindex_vectors_1', - origin: 'pk', - partial: 0, - unique: 1, - }, - { - columns: [ - {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, - {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'symbol_id'}, - {cid: 2, coll: 'BINARY', desc: 0, key: 1, name: 'fingerprint'}, - ], - name: 'vector_reuse_lookup', - origin: 'c', - partial: 0, - unique: 0, - }, -] as const satisfies readonly ExpectedVectorIndex[]; - -const VECTOR_RETIREMENT_STATE_INDEXES = [ - { - columns: [ - {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'singleton'}, - {cid: 1, coll: 'BINARY', desc: 0, key: 0, name: 'admission_cursor'}, - {cid: 2, coll: 'BINARY', desc: 0, key: 0, name: 'generation_revision'}, - {cid: 3, coll: 'BINARY', desc: 0, key: 0, name: 'admission_scan_revision'}, - {cid: 4, coll: 'BINARY', desc: 0, key: 0, name: 'clean_generation_revision'}, - {cid: 5, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_worktree_id'}, - {cid: 6, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_generation'}, - {cid: 7, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_snapshot_id'}, - ], - name: 'sqlite_autoindex_vector_retirement_state_1', - origin: 'pk', - partial: 0, - unique: 1, - }, -] as const satisfies readonly ExpectedVectorIndex[]; - -const VECTOR_RETIREMENT_MARKER_INDEXES = [ - { - columns: [{cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], - name: 'sqlite_autoindex_vector_generation_retirements_1', - origin: 'u', - partial: 0, - unique: 1, - }, - { - columns: [ - {cid: 3, coll: 'BINARY', desc: 0, key: 1, name: 'retired_by_worktree_id'}, - {cid: 2, coll: 'BINARY', desc: 0, key: 1, name: 'snapshot_id'}, - {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, - {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'retirement_id'}, - rowIdPayload, - ], - name: 'vector_generation_retirement_association', - origin: 'c', - partial: 1, - unique: 0, - }, -] as const satisfies readonly ExpectedVectorIndex[]; - -const VECTOR_POINTER_FOREIGN_KEYS = [ - { - from: 'generation', - id: 0, - match: 'NONE', - onDelete: 'CASCADE', - onUpdate: 'NO ACTION', - seq: 0, - table: 'vector_generations', - to: 'generation', - }, -] as const satisfies readonly ExpectedVectorForeignKey[]; - -const VECTOR_ROW_FOREIGN_KEYS = VECTOR_POINTER_FOREIGN_KEYS; - -const boundedVectorUserTableNames = Effect.fn('codeGraph.boundedVectorUserTableNames')(function* ( - sql: SqlClient.SqlClient, -) { - const rows = yield* sql.unsafe<{readonly name: unknown}>( - `SELECT CASE - WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL - END AS name - FROM sqlite_master - WHERE type = 'table' AND name NOT GLOB 'sqlite_*' - LIMIT ?`, - [VECTOR_CORE_TABLE_NAMES.length + VECTOR_RETIREMENT_TABLE_NAMES.length + 1], - ); - if (rows.some(row => typeof row.name !== 'string')) return undefined; - return rows.map(row => String(row.name)); -}); - -function sameStringSet(observed: readonly string[] | undefined, expected: readonly string[]): boolean { - return ( - observed !== undefined && - observed.length === expected.length && - [...observed].sort().every((value, index) => value === [...expected].sort()[index]) - ); -} - -const exactVectorIndexSet = Effect.fn('codeGraph.exactVectorIndexSet')(function* ( - sql: SqlClient.SqlClient, - tableName: string, - expected: readonly ExpectedVectorIndex[], -) { - const rows = yield* sql.unsafe<{ - readonly name: unknown; - readonly origin: unknown; - readonly partial: unknown; - readonly unique_value: unknown; - }>( - `SELECT - CASE WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL END AS name, - CASE WHEN origin IN ('c', 'pk', 'u') THEN origin ELSE NULL END AS origin, - partial, - "unique" AS unique_value - FROM pragma_index_list(${sqliteStringLiteral(tableName)}) - LIMIT ?`, - [expected.length + 1], - ); - if (rows.length !== expected.length) return false; - const byName = [...expected].sort((left, right) => left.name.localeCompare(right.name)); - const observed = [...rows].sort((left, right) => String(left.name).localeCompare(String(right.name))); - for (let index = 0; index < byName.length; index += 1) { - const definition = byName[index]!; - const row = observed[index]; - if ( - row?.name !== definition.name || - row.origin !== definition.origin || - row.partial !== definition.partial || - row.unique_value !== definition.unique - ) { - return false; - } - const columns = yield* sql.unsafe<{ - readonly cid: unknown; - readonly coll: unknown; - readonly desc_value: unknown; - readonly key_value: unknown; - readonly name: unknown; - readonly seqno: unknown; - }>( - `SELECT seqno, cid, - CASE WHEN name IS NULL THEN NULL - WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name - ELSE 0 END AS name, - "desc" AS desc_value, - CASE WHEN coll = 'BINARY' THEN coll ELSE NULL END AS coll, - "key" AS key_value - FROM pragma_index_xinfo(${sqliteStringLiteral(definition.name)}) - LIMIT ?`, - [definition.columns.length + 1], - ); - if ( - columns.length !== definition.columns.length || - columns.some((column, columnIndex) => { - const expectedColumn = definition.columns[columnIndex]; - return ( - column.seqno !== columnIndex || - column.cid !== expectedColumn?.cid || - column.name !== expectedColumn.name || - column.desc_value !== expectedColumn.desc || - column.coll !== expectedColumn.coll || - column.key_value !== expectedColumn.key - ); - }) - ) { - return false; - } - } - return true; -}); - -const exactVectorForeignKeys = Effect.fn('codeGraph.exactVectorForeignKeys')(function* ( - sql: SqlClient.SqlClient, - tableName: string, - expected: readonly ExpectedVectorForeignKey[], -) { - const rows = yield* sql.unsafe<{ - readonly from_column: unknown; - readonly id: unknown; - readonly match_value: unknown; - readonly on_delete: unknown; - readonly on_update: unknown; - readonly seq: unknown; - readonly table_name: unknown; - readonly to_column: unknown; - }>( - `SELECT id, seq, - CASE WHEN typeof("table") = 'text' AND length(CAST("table" AS BLOB)) <= 128 - THEN "table" ELSE NULL END AS table_name, - CASE WHEN typeof("from") = 'text' AND length(CAST("from" AS BLOB)) <= 128 - THEN "from" ELSE NULL END AS from_column, - CASE WHEN typeof("to") = 'text' AND length(CAST("to" AS BLOB)) <= 128 - THEN "to" ELSE NULL END AS to_column, - on_update, on_delete, "match" AS match_value - FROM pragma_foreign_key_list(${sqliteStringLiteral(tableName)}) - LIMIT ?`, - [expected.length + 1], - ); - return ( - rows.length === expected.length && - rows.every((row, index) => { - const definition = expected[index]; - return ( - row.id === definition?.id && - row.seq === definition.seq && - row.table_name === definition.table && - row.from_column === definition.from && - row.to_column === definition.to && - row.on_update === definition.onUpdate && - row.on_delete === definition.onDelete && - row.match_value === definition.match - ); - }) - ); -}); - -const codeGraphVectorCoreSchemaState = Effect.fn('codeGraph.vectorCoreSchemaState')(function* ( - sql: SqlClient.SqlClient, -) { - const expected = [ - {name: 'vector_generations', sql: CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL, type: 'table'}, - {name: 'vector_pointers', sql: CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL, type: 'table'}, - {name: 'vectors', sql: CODE_GRAPH_VECTORS_TABLE_SQL, type: 'table'}, - {name: 'vector_reuse_lookup', sql: CODE_GRAPH_VECTOR_REUSE_INDEX_SQL, type: 'index'}, - ] as const; - for (const object of expected) { - const rows = yield* boundedSchemaObjects(sql, object.name, 2); - if ( - rows.length !== 1 || - rows[0]?.name !== object.name || - rows[0]?.type !== object.type || - normalizeSchemaDefinition(String(rows[0]?.sql ?? '')) !== normalizeSchemaDefinition(object.sql) - ) { - return 'incompatible' as const; - } - } - const userTables = yield* boundedVectorUserTableNames(sql); - const allowedTables = new Set([...VECTOR_CORE_TABLE_NAMES, ...VECTOR_RETIREMENT_TABLE_NAMES]); - if ( - userTables === undefined || - VECTOR_CORE_TABLE_NAMES.some(name => !userTables.includes(name)) || - userTables.some(name => !allowedTables.has(name)) - ) { - return 'incompatible' as const; - } - if ( - !(yield* exactVectorIndexSet(sql, 'vector_generations', VECTOR_GENERATION_INDEXES)) || - !(yield* exactVectorIndexSet(sql, 'vectors', VECTOR_ROW_INDEXES)) || - !(yield* exactVectorForeignKeys(sql, 'vector_generations', [])) || - !(yield* exactVectorForeignKeys(sql, 'vector_pointers', VECTOR_POINTER_FOREIGN_KEYS)) || - !(yield* exactVectorForeignKeys(sql, 'vectors', VECTOR_ROW_FOREIGN_KEYS)) - ) { - return 'incompatible' as const; - } - const pointerIndexesReady = yield* exactVectorIndexSet(sql, 'vector_pointers', VECTOR_POINTER_INDEXES); - const pointerIndexesLegacy = yield* exactVectorIndexSet( - sql, - 'vector_pointers', - VECTOR_POINTER_INDEXES_WITHOUT_GENERATION, - ); - if (!pointerIndexesReady && !pointerIndexesLegacy) return 'incompatible' as const; - const pointerIndex = yield* boundedSchemaObjects(sql, 'vector_pointer_generation_lookup', 2); - if (pointerIndex.length === 0) { - return pointerIndexesLegacy ? ('missing-pointer-index' as const) : ('incompatible' as const); - } - if ( - pointerIndex.length !== 1 || - pointerIndex[0]?.name !== 'vector_pointer_generation_lookup' || - pointerIndex[0]?.type !== 'index' || - normalizeSchemaDefinition(String(pointerIndex[0]?.sql ?? '')) !== - normalizeSchemaDefinition(CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL) - ) { - return 'incompatible' as const; - } - return pointerIndexesReady ? ('ready' as const) : ('incompatible' as const); -}); - -const codeGraphVectorCoreSchemaCurrent = Effect.fn('codeGraph.vectorCoreSchemaCurrent')(function* ( - sql: SqlClient.SqlClient, -) { - return (yield* codeGraphVectorCoreSchemaState(sql)) === 'ready'; -}); - -const inspectLegacyPointerIndexPlan = Effect.fn('codeGraph.inspectLegacyVectorPointerIndexPlan')(function* ( - sql: SqlClient.SqlClient, -) { - const statement = codeGraphVectorRetirementLegacyPointerProbeStatement(); - const observed = yield* sql.unsafe<{ - readonly generation: unknown; - readonly identity_bytes: unknown; - readonly worktree_id: unknown; - }>(statement.text, statement.parameters); - if (observed.length > CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS) { - return yield* Effect.fail(new Error('Code graph vector pointer index exceeds its bounded migration limit.')); - } - let finalFactBytes = 0; - const rows: Array<{readonly generation: string; readonly worktreeId: string}> = []; - for (const row of observed) { - if ( - typeof row.worktree_id !== 'string' || - !/^[0-9a-f]{64}$/.test(row.worktree_id) || - typeof row.generation !== 'string' || - !validBoundedText(row.generation, VECTOR_GENERATION_BYTES) || - !Number.isSafeInteger(row.identity_bytes) || - Number(row.identity_bytes) <= 64 - ) { - return yield* Effect.fail(new Error('Code graph vector pointer index manifest is invalid.')); - } - finalFactBytes += Number(row.identity_bytes); - if (!Number.isSafeInteger(finalFactBytes) || finalFactBytes > CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES) { - return yield* Effect.fail(new Error('Code graph vector pointer index exceeds its bounded byte limit.')); - } - rows.push({generation: row.generation, worktreeId: row.worktree_id}); - } - return {finalFactBytes, rows, storage: yield* inspectVectorPageStorageSql(sql)} satisfies LegacyPointerIndexPlan; -}); - -function sameLegacyPointerIndexPlan(left: LegacyPointerIndexPlan, right: LegacyPointerIndexPlan): boolean { - return ( - left.finalFactBytes === right.finalFactBytes && - sameVectorPageStorage(left.storage, right.storage) && - left.rows.length === right.rows.length && - left.rows.every( - (row, index) => - row.worktreeId === right.rows[index]?.worktreeId && row.generation === right.rows[index]?.generation, - ) - ); -} - -const codeGraphVectorRetirementSchemaState = Effect.fn('codeGraph.vectorRetirementSchemaState')(function* ( - sql: SqlClient.SqlClient, -) { - const expected = [ - {name: 'vector_retirement_state', sql: CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, type: 'table'}, - {name: 'vector_generation_retirements', sql: CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, type: 'table'}, - { - name: 'vector_generation_retirement_association', - sql: CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, - type: 'index', - }, - ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map(trigger => ({ - name: trigger.name, - sql: trigger.sql, - type: 'trigger' as const, - })), - ]; - const observed: Array<'absent' | 'current' | 'incompatible'> = []; - for (const object of expected) { - const rows = yield* boundedSchemaObjects(sql, object.name, 2); - if (rows.length === 0) { - observed.push('absent'); - continue; - } - observed.push( - rows.length === 1 && - rows[0]?.name === object.name && - rows[0]?.type === object.type && - normalizeSchemaDefinition(String(rows[0]?.sql ?? '')) === normalizeSchemaDefinition(object.sql) - ? 'current' - : 'incompatible', - ); - } - const triggerRows = yield* sql.unsafe<{readonly name: unknown; readonly tbl_name: unknown}>( - `SELECT - CASE WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL END AS name, - CASE WHEN typeof(tbl_name) = 'text' AND length(CAST(tbl_name AS BLOB)) <= 64 THEN tbl_name ELSE NULL END AS tbl_name - FROM sqlite_master - WHERE type = 'trigger' - AND tbl_name COLLATE NOCASE IN ( - 'vector_retirement_state', - 'vector_generation_retirements', - 'vector_generations', - 'vector_pointers', - 'vectors' - ) - LIMIT ?`, - [CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.length + 1], - ); - if (observed.every(state => state === 'absent')) { - const userTables = yield* boundedVectorUserTableNames(sql); - const sequenceTable = yield* boundedSchemaObjects(sql, 'sqlite_sequence', 2); - return triggerRows.length === 0 && sequenceTable.length === 0 && sameStringSet(userTables, VECTOR_CORE_TABLE_NAMES) - ? ('absent' as const) - : ('incompatible' as const); - } - if (!observed.every(state => state === 'current')) return 'incompatible' as const; - const userTables = yield* boundedVectorUserTableNames(sql); - if ( - !sameStringSet(userTables, [...VECTOR_CORE_TABLE_NAMES, ...VECTOR_RETIREMENT_TABLE_NAMES]) || - !(yield* exactVectorIndexSet(sql, 'vector_retirement_state', VECTOR_RETIREMENT_STATE_INDEXES)) || - !(yield* exactVectorIndexSet(sql, 'vector_generation_retirements', VECTOR_RETIREMENT_MARKER_INDEXES)) || - !(yield* exactVectorForeignKeys(sql, 'vector_retirement_state', [])) || - !(yield* exactVectorForeignKeys(sql, 'vector_generation_retirements', [])) - ) { - return 'incompatible' as const; - } - const expectedTriggerNames = [ - ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map( - trigger => `${trigger.name}\0${vectorRetirementTriggerTarget(trigger.name)}`, - ), - ].sort(); - const observedTriggerNames = triggerRows - .map(row => - typeof row.name === 'string' && typeof row.tbl_name === 'string' ? `${row.name}\0${row.tbl_name}` : '', - ) - .sort(); - if ( - observedTriggerNames.length !== expectedTriggerNames.length || - observedTriggerNames.some((name, index) => name !== expectedTriggerNames[index]) - ) { - return 'incompatible' as const; - } - const stateRows = yield* sql.unsafe<{ - readonly admission_cursor: unknown; - readonly admission_scan_revision: unknown; - readonly clean_generation_revision: unknown; - readonly generation_revision: unknown; - readonly pointer_delete_present: unknown; - readonly singleton: unknown; - }>( - `SELECT singleton, - CASE - WHEN admission_cursor IS NULL THEN NULL - WHEN typeof(admission_cursor) = 'text' - AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(admission_cursor, char(0)) = 0 - THEN admission_cursor ELSE 0 - END AS admission_cursor, - CASE - WHEN typeof(generation_revision) = 'integer' - AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN generation_revision ELSE NULL - END AS generation_revision, - CASE - WHEN admission_scan_revision IS NULL THEN NULL - WHEN typeof(admission_scan_revision) = 'integer' - AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN admission_scan_revision ELSE -1 - END AS admission_scan_revision, - CASE - WHEN clean_generation_revision IS NULL THEN NULL - WHEN typeof(clean_generation_revision) = 'integer' - AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN clean_generation_revision ELSE -1 - END AS clean_generation_revision, - CASE - WHEN pointer_delete_worktree_id IS NULL - AND pointer_delete_generation IS NULL - AND pointer_delete_snapshot_id IS NULL - THEN 0 ELSE 1 - END AS pointer_delete_present - FROM vector_retirement_state LIMIT 2`, - ); - if ( - stateRows.length !== 1 || - stateRows[0]?.singleton !== 1 || - !Number.isSafeInteger(stateRows[0]?.generation_revision) || - Number(stateRows[0]?.generation_revision) < 0 || - (stateRows[0]?.admission_scan_revision !== null && - (!Number.isSafeInteger(stateRows[0]?.admission_scan_revision) || - Number(stateRows[0]?.admission_scan_revision) < 0 || - Number(stateRows[0]?.admission_scan_revision) > Number(stateRows[0]?.generation_revision))) || - (stateRows[0]?.clean_generation_revision !== null && - (!Number.isSafeInteger(stateRows[0]?.clean_generation_revision) || - Number(stateRows[0]?.clean_generation_revision) < 0 || - Number(stateRows[0]?.clean_generation_revision) > Number(stateRows[0]?.generation_revision))) || - (stateRows[0]?.admission_cursor === null) !== (stateRows[0]?.admission_scan_revision === null) || - (stateRows[0]?.admission_scan_revision !== null && - stateRows[0]?.clean_generation_revision !== null && - Number(stateRows[0]?.clean_generation_revision) > Number(stateRows[0]?.admission_scan_revision)) || - (stateRows[0]?.clean_generation_revision !== null && - Number(stateRows[0]?.clean_generation_revision) === Number(stateRows[0]?.generation_revision) && - (stateRows[0]?.admission_cursor !== null || stateRows[0]?.admission_scan_revision !== null)) || - stateRows[0]?.pointer_delete_present !== 0 || - (stateRows[0]?.admission_cursor !== null && - (typeof stateRows[0]?.admission_cursor !== 'string' || - !validBoundedText(stateRows[0].admission_cursor, VECTOR_GENERATION_BYTES))) - ) { - return 'incompatible' as const; - } - const sequenceRows = yield* sql.unsafe<{ - readonly name: unknown; - readonly seq: unknown; - readonly seq_type: unknown; - }>( - `SELECT - CASE WHEN typeof(name) = 'text' - AND name = 'vector_generation_retirements' - THEN name ELSE NULL END AS name, - CASE - WHEN typeof(seq) = 'integer' AND seq BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN seq ELSE NULL - END AS seq, - typeof(seq) AS seq_type - FROM sqlite_sequence - WHERE name = 'vector_generation_retirements' COLLATE NOCASE - LIMIT 2`, - ); - const maximumRows = yield* sql.unsafe<{readonly maximum: unknown}>( - `SELECT CASE - WHEN typeof(retirement_id) = 'integer' - AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN retirement_id ELSE NULL - END AS maximum - FROM vector_generation_retirements - ORDER BY retirement_id DESC - LIMIT 1`, - ); - const maximum = maximumRows.length === 0 ? null : maximumRows[0]?.maximum; - if ( - sequenceRows.length !== 1 || - sequenceRows[0]?.name !== 'vector_generation_retirements' || - sequenceRows[0]?.seq_type !== 'integer' || - !Number.isSafeInteger(sequenceRows[0]?.seq) || - Number(sequenceRows[0]?.seq) < 0 || - (maximum !== null && - (!Number.isSafeInteger(maximum) || Number(maximum) <= 0 || Number(sequenceRows[0]?.seq) < Number(maximum))) - ) { - return 'incompatible' as const; - } - return 'ready' as const; -}); - -const selectVectorRetirementMarker = Effect.fn('codeGraph.selectVectorRetirementMarker')(function* ( - sql: SqlClient.SqlClient, - generation: string, -) { - const rows = yield* sql.unsafe<{ - readonly delete_authorized: unknown; - readonly generation: unknown; - readonly page_revision: unknown; - readonly retired_by_worktree_id: unknown; - readonly retirement_id: unknown; - readonly snapshot_id: unknown; - }>( - `SELECT - CASE - WHEN typeof(retirement_id) = 'integer' - AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN retirement_id ELSE NULL - END AS retirement_id, - CASE - WHEN typeof(generation) = 'text' - AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} - AND instr(generation, char(0)) = 0 - THEN generation ELSE NULL - END AS generation, - CASE - WHEN typeof(snapshot_id) = 'text' - AND length(CAST(snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} - AND instr(snapshot_id, char(0)) = 0 - THEN snapshot_id ELSE NULL - END AS snapshot_id, - CASE - WHEN retired_by_worktree_id IS NULL THEN NULL - WHEN typeof(retired_by_worktree_id) = 'text' - AND length(CAST(retired_by_worktree_id AS BLOB)) = 64 - AND retired_by_worktree_id NOT GLOB '*[^0-9a-f]*' - THEN retired_by_worktree_id ELSE 0 - END AS retired_by_worktree_id, - CASE - WHEN typeof(page_revision) = 'integer' - AND page_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} - THEN page_revision ELSE NULL - END AS page_revision, - CASE - WHEN typeof(delete_authorized) = 'integer' AND delete_authorized IN (0, 1) - THEN delete_authorized ELSE NULL - END AS delete_authorized - FROM vector_generation_retirements - WHERE generation = ? LIMIT 2`, - [generation], - ); - if (rows.length === 0) return undefined; - const row = rows[0]; - if ( - rows.length !== 1 || - !Number.isSafeInteger(row?.retirement_id) || - Number(row?.retirement_id) <= 0 || - typeof row?.generation !== 'string' || - !validBoundedText(row.generation, VECTOR_GENERATION_BYTES) || - typeof row?.snapshot_id !== 'string' || - !validBoundedText(row.snapshot_id, VECTOR_SNAPSHOT_BYTES) || - (row?.retired_by_worktree_id !== null && - (typeof row?.retired_by_worktree_id !== 'string' || !/^[0-9a-f]{64}$/.test(row.retired_by_worktree_id))) || - !Number.isSafeInteger(row?.page_revision) || - Number(row?.page_revision) < 0 || - (row?.delete_authorized !== 0 && row?.delete_authorized !== 1) - ) { - return yield* Effect.fail(new Error('Code graph vector retirement marker is invalid.')); - } - return { - deleteAuthorized: row.delete_authorized === 1, - generation: row.generation, - pageRevision: Number(row.page_revision), - ...(typeof row.retired_by_worktree_id === 'string' ? {retiredByWorktreeId: row.retired_by_worktree_id} : {}), - retirementId: Number(row.retirement_id), - snapshotId: row.snapshot_id, - } satisfies CodeGraphVectorRetirementMarker; -}); - -export const selectCodeGraphVectorRetirementMarker = selectVectorRetirementMarker; - -function useExistingVectorDatabase( - databasePath: string, - effect: Effect.Effect, -): Effect.Effect> { - return Effect.scoped( - effect.pipe( - Effect.provide( - SqliteClient.layer({ - create: false, - disableWAL: true, - filename: databasePath, - readwrite: true, - }), - ), - ), - ) as Effect.Effect>; -} - -function vectorRetirementTriggerTarget( - name: (typeof CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS)[number]['name'], -): string { - if (name.includes('_marker_')) return 'vector_generation_retirements'; - if (name.includes('_pointer_')) return 'vector_pointers'; - if (name.includes('_generation_')) return 'vector_generations'; - return 'vectors'; -} - -function useReadOnlyVectorDatabase( - databasePath: string, - effect: Effect.Effect, -): Effect.Effect> { - return Effect.scoped( - effect.pipe( - Effect.provide( - SqliteClient.layer({ - create: false, - disableWAL: true, - filename: databasePath, - readonly: true, - readwrite: false, - }), - ), - ), - ) as Effect.Effect>; -} - -/** @internal Read-only frozen pager tuple for a separately protected cursor publication. */ -export const inspectCodeGraphVectorPageStorage = Effect.fn('codeGraph.inspectVectorPageStorage')(function* ( - databasePath: string, -) { - return yield* useReadOnlyVectorDatabase( - databasePath, - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - return yield* inspectVectorPageStorageSql(sql); - }), - ); -}); - -const inspectVectorPageStorageSql = Effect.fn('codeGraph.inspectVectorPageStorageSql')(function* ( - sql: SqlClient.SqlClient, -) { - const [pageSizeRows, freelistRows, walRows, journalRows] = yield* Effect.all( - [ - sql.unsafe<{readonly page_size: unknown}>('PRAGMA page_size'), - sql.unsafe<{readonly freelist_count: unknown}>('PRAGMA freelist_count'), - sql.unsafe<{readonly wal_autocheckpoint: unknown}>('PRAGMA wal_autocheckpoint'), - sql.unsafe<{readonly journal_mode: unknown}>('PRAGMA journal_mode'), - ] as const, - {concurrency: 1}, - ); - const pageSize = pageSizeRows[0]?.page_size; - const freelistPages = freelistRows[0]?.freelist_count; - const walAutoCheckpointPages = walRows[0]?.wal_autocheckpoint; - const journalMode = journalRows[0]?.journal_mode; - if ( - !Number.isSafeInteger(pageSize) || - Number(pageSize) <= 0 || - !Number.isSafeInteger(freelistPages) || - Number(freelistPages) < 0 || - !Number.isSafeInteger(walAutoCheckpointPages) || - Number(walAutoCheckpointPages) <= 0 || - (journalMode !== 'delete' && journalMode !== 'wal') - ) { - return yield* Effect.fail(new Error('Code graph vector page storage is invalid.')); - } - const freelistBytes = Number(pageSize) * Number(freelistPages); - if (!Number.isSafeInteger(freelistBytes)) { - return yield* Effect.fail(new Error('Code graph vector page storage is invalid.')); - } - return { - freelistBytes, - journalMode, - pageSize: Number(pageSize), - walAutoCheckpointPages: Number(walAutoCheckpointPages), - } as const satisfies CodeGraphVectorPageStorage; -}); - -function sameVectorPageStorage(left: CodeGraphVectorPageStorage, right: CodeGraphVectorPageStorage): boolean { - return ( - left.freelistBytes === right.freelistBytes && - left.pageSize === right.pageSize && - left.walAutoCheckpointPages === right.walAutoCheckpointPages && - left.journalMode === right.journalMode - ); -} - -function vectorRetirementPageAuthorityBytes(marker: CodeGraphVectorRetirementMarker): number { - return ( - new TextEncoder().encode(marker.generation).byteLength + - new TextEncoder().encode(marker.snapshotId).byteLength + - (marker.retiredByWorktreeId === undefined ? 0 : 64) + - 256 - ); -} - -function boundedRetirementLimit(requestedLimit: number): number { - if (!Number.isSafeInteger(requestedLimit) || requestedLimit <= 0) { - throw new Error('Code graph vector retirement page limit is invalid.'); - } - return Math.min(requestedLimit, CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS); -} - -function validBoundedText(value: string, maximumBytes: number): boolean { - return ( - typeof value === 'string' && - value.length > 0 && - !value.includes('\0') && - new TextEncoder().encode(value).byteLength <= maximumBytes - ); -} - -const boundedSchemaObjects = Effect.fn('codeGraph.boundedVectorSchemaObjects')(function* ( - sql: SqlClient.SqlClient, - name: string, - limit: number, -) { - return yield* sql.unsafe<{ - readonly name: unknown; - readonly sql: unknown; - readonly type: unknown; - }>( - `SELECT name, type, - CASE WHEN typeof(sql) = 'text' AND length(CAST(sql AS BLOB)) <= ? THEN sql ELSE NULL END AS sql - FROM sqlite_master - WHERE name = ? COLLATE NOCASE - LIMIT ?`, - [VECTOR_RETIREMENT_TRIGGER_SQL_BYTES, name, limit], - ); -}); - -function normalizeSchemaDefinition(value: string): string { - const quoted: string[] = []; - let unquoted = ''; - for (let index = 0; index < value.length; index += 1) { - const opener = value[index]!; - const closer = opener === '[' ? ']' : opener; - if (opener !== "'" && opener !== '"' && opener !== '`' && opener !== '[') { - unquoted += opener; - continue; - } - const start = index; - for (index += 1; index < value.length; index += 1) { - if (value[index] !== closer) continue; - if (closer !== ']' && value[index + 1] === closer) { - index += 1; - continue; - } - break; - } - quoted.push(value.slice(start, Math.min(index + 1, value.length))); - unquoted += `\u0000${quoted.length - 1}\u0000`; - } - return unquoted - .toLowerCase() - .replace(/\bif not exists\b/gu, '') - .replace(/\s+/gu, ' ') - .replace(/\s*([(),])\s*/gu, '$1') - .trim() - .split('\u0000') - .map((segment, index) => (index % 2 === 1 ? (quoted[Number(segment)] ?? '') : segment)) - .join(''); -} - -function storedSchemaSql(value: string): string { - return value.replace(/^CREATE (TABLE|INDEX|TRIGGER) IF NOT EXISTS/u, 'CREATE $1'); -} - -function sqliteStringLiteral(value: string): string { - return `'${value.replaceAll("'", "''")}'`; -} - -const lastStatementChangeCount = Effect.fn('codeGraph.vectorRetirementChangeCount')(function* ( - sql: SqlClient.SqlClient, -) { - const rows = yield* sql.unsafe<{readonly count: unknown}>('SELECT changes() AS count'); - const count = rows[0]?.count; - if (!Number.isSafeInteger(count) || Number(count) < 0) { - return yield* Effect.fail(new Error('Code graph vector retirement change count is invalid.')); - } - return Number(count); -}); diff --git a/src/code_graph/vector_retirement_inspection.ts b/src/code_graph/vector_retirement_inspection.ts new file mode 100644 index 00000000..443279c5 --- /dev/null +++ b/src/code_graph/vector_retirement_inspection.ts @@ -0,0 +1,941 @@ +import * as SqliteClient from '@effect/sql-sqlite-bun/SqliteClient'; +import {Effect, Layer} from 'effect'; +import * as SqlClient from 'effect/unstable/sql/SqlClient'; +import { + CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL, + CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL, + CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL, + CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES, + CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS, + CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS, + CODE_GRAPH_VECTOR_REUSE_INDEX_SQL, + CODE_GRAPH_VECTORS_TABLE_SQL, + CodeGraphVectorRetirementError, + MAXIMUM_SAFE_INTEGER_SQL, + VECTOR_CORE_TABLE_NAMES, + VECTOR_GENERATION_BYTES, + VECTOR_RETIREMENT_TABLE_NAMES, + VECTOR_RETIREMENT_TRIGGER_SQL_BYTES, + VECTOR_SNAPSHOT_BYTES, + sqliteStringLiteral, +} from './vector_retirement_schema.js'; + +export interface CodeGraphVectorRetirementMarker { + readonly deleteAuthorized: boolean; + readonly generation: string; + readonly pageRevision: number; + readonly retiredByWorktreeId?: string; + readonly retirementId: number; + readonly snapshotId: string; +} + +export interface CodeGraphVectorPageStorage { + readonly freelistBytes: number; + readonly journalMode: 'delete' | 'wal'; + readonly pageSize: number; + readonly walAutoCheckpointPages: number; +} + +/** @internal Frozen manifest for the released-v2 pointer-index bridge. */ +export interface LegacyPointerIndexPlan { + readonly finalFactBytes: number; + readonly rows: readonly {readonly generation: string; readonly worktreeId: string}[]; + readonly storage: CodeGraphVectorPageStorage; +} + +export function codeGraphVectorRetirementLegacyPointerProbeStatement() { + return { + parameters: [CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS + 1] as const, + text: `SELECT + CASE + WHEN typeof(worktree_id) = 'text' + AND length(CAST(worktree_id AS BLOB)) = 64 + AND worktree_id NOT GLOB '*[^0-9a-f]*' + THEN worktree_id ELSE NULL + END AS worktree_id, + CASE + WHEN typeof(generation) = 'text' + AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(generation, char(0)) = 0 + THEN generation ELSE NULL + END AS generation, + length(CAST(worktree_id AS BLOB)) + length(CAST(generation AS BLOB)) AS identity_bytes + FROM vector_pointers + ORDER BY vector_pointers.worktree_id + LIMIT ?`, + }; +} + +export interface ExpectedVectorIndexColumn { + readonly cid: number; + readonly coll: 'BINARY'; + readonly desc: 0; + readonly key: 0 | 1; + readonly name: string | null; +} + +export interface ExpectedVectorIndex { + readonly columns: readonly ExpectedVectorIndexColumn[]; + readonly name: string; + readonly origin: 'c' | 'pk' | 'u'; + readonly partial: 0 | 1; + readonly unique: 0 | 1; +} + +export interface ExpectedVectorForeignKey { + readonly from: string; + readonly id: number; + readonly match: 'NONE'; + readonly onDelete: 'CASCADE' | 'NO ACTION'; + readonly onUpdate: 'NO ACTION'; + readonly seq: number; + readonly table: string; + readonly to: string; +} + +export const rowIdPayload = {cid: -1, coll: 'BINARY', desc: 0, key: 0, name: null} as const; + +export const VECTOR_GENERATION_INDEXES = [ + { + columns: [{cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], + name: 'sqlite_autoindex_vector_generations_1', + origin: 'pk', + partial: 0, + unique: 1, + }, +] as const satisfies readonly ExpectedVectorIndex[]; + +export const VECTOR_POINTER_PRIMARY_INDEX = { + columns: [{cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'worktree_id'}, rowIdPayload], + name: 'sqlite_autoindex_vector_pointers_1', + origin: 'pk', + partial: 0, + unique: 1, +} as const satisfies ExpectedVectorIndex; + +export const VECTOR_POINTER_INDEXES_WITHOUT_GENERATION = [VECTOR_POINTER_PRIMARY_INDEX] as const; +export const VECTOR_POINTER_INDEXES = [ + VECTOR_POINTER_PRIMARY_INDEX, + { + columns: [{cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], + name: 'vector_pointer_generation_lookup', + origin: 'c', + partial: 0, + unique: 0, + }, +] as const satisfies readonly ExpectedVectorIndex[]; + +export const VECTOR_ROW_INDEXES = [ + { + columns: [ + {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, + {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'symbol_id'}, + {cid: 2, coll: 'BINARY', desc: 0, key: 0, name: 'fingerprint'}, + {cid: 3, coll: 'BINARY', desc: 0, key: 0, name: 'vector'}, + ], + name: 'sqlite_autoindex_vectors_1', + origin: 'pk', + partial: 0, + unique: 1, + }, + { + columns: [ + {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, + {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'symbol_id'}, + {cid: 2, coll: 'BINARY', desc: 0, key: 1, name: 'fingerprint'}, + ], + name: 'vector_reuse_lookup', + origin: 'c', + partial: 0, + unique: 0, + }, +] as const satisfies readonly ExpectedVectorIndex[]; + +export const VECTOR_RETIREMENT_STATE_INDEXES = [ + { + columns: [ + {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'singleton'}, + {cid: 1, coll: 'BINARY', desc: 0, key: 0, name: 'admission_cursor'}, + {cid: 2, coll: 'BINARY', desc: 0, key: 0, name: 'generation_revision'}, + {cid: 3, coll: 'BINARY', desc: 0, key: 0, name: 'admission_scan_revision'}, + {cid: 4, coll: 'BINARY', desc: 0, key: 0, name: 'clean_generation_revision'}, + {cid: 5, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_worktree_id'}, + {cid: 6, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_generation'}, + {cid: 7, coll: 'BINARY', desc: 0, key: 0, name: 'pointer_delete_snapshot_id'}, + ], + name: 'sqlite_autoindex_vector_retirement_state_1', + origin: 'pk', + partial: 0, + unique: 1, + }, +] as const satisfies readonly ExpectedVectorIndex[]; + +export const VECTOR_RETIREMENT_MARKER_INDEXES = [ + { + columns: [{cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, rowIdPayload], + name: 'sqlite_autoindex_vector_generation_retirements_1', + origin: 'u', + partial: 0, + unique: 1, + }, + { + columns: [ + {cid: 3, coll: 'BINARY', desc: 0, key: 1, name: 'retired_by_worktree_id'}, + {cid: 2, coll: 'BINARY', desc: 0, key: 1, name: 'snapshot_id'}, + {cid: 1, coll: 'BINARY', desc: 0, key: 1, name: 'generation'}, + {cid: 0, coll: 'BINARY', desc: 0, key: 1, name: 'retirement_id'}, + rowIdPayload, + ], + name: 'vector_generation_retirement_association', + origin: 'c', + partial: 1, + unique: 0, + }, +] as const satisfies readonly ExpectedVectorIndex[]; + +export const VECTOR_POINTER_FOREIGN_KEYS = [ + { + from: 'generation', + id: 0, + match: 'NONE', + onDelete: 'CASCADE', + onUpdate: 'NO ACTION', + seq: 0, + table: 'vector_generations', + to: 'generation', + }, +] as const satisfies readonly ExpectedVectorForeignKey[]; + +export const VECTOR_ROW_FOREIGN_KEYS = VECTOR_POINTER_FOREIGN_KEYS; + +export const boundedVectorUserTableNames = Effect.fn('codeGraph.boundedVectorUserTableNames')(function* ( + sql: SqlClient.SqlClient, +) { + const rows = yield* sql.unsafe<{readonly name: unknown}>( + `SELECT CASE + WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL + END AS name + FROM sqlite_master + WHERE type = 'table' AND name NOT GLOB 'sqlite_*' + LIMIT ?`, + [VECTOR_CORE_TABLE_NAMES.length + VECTOR_RETIREMENT_TABLE_NAMES.length + 1], + ); + if (rows.some(row => typeof row.name !== 'string')) return undefined; + return rows.map(row => String(row.name)); +}); + +export function sameStringSet(observed: readonly string[] | undefined, expected: readonly string[]): boolean { + return ( + observed !== undefined && + observed.length === expected.length && + [...observed].sort().every((value, index) => value === [...expected].sort()[index]) + ); +} + +export const exactVectorIndexSet = Effect.fn('codeGraph.exactVectorIndexSet')(function* ( + sql: SqlClient.SqlClient, + tableName: string, + expected: readonly ExpectedVectorIndex[], +) { + const rows = yield* sql.unsafe<{ + readonly name: unknown; + readonly origin: unknown; + readonly partial: unknown; + readonly unique_value: unknown; + }>( + `SELECT + CASE WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL END AS name, + CASE WHEN origin IN ('c', 'pk', 'u') THEN origin ELSE NULL END AS origin, + partial, + "unique" AS unique_value + FROM pragma_index_list(${sqliteStringLiteral(tableName)}) + LIMIT ?`, + [expected.length + 1], + ); + if (rows.length !== expected.length) return false; + const byName = [...expected].sort((left, right) => left.name.localeCompare(right.name)); + const observed = [...rows].sort((left, right) => String(left.name).localeCompare(String(right.name))); + for (let index = 0; index < byName.length; index += 1) { + const definition = byName[index]!; + const row = observed[index]; + if ( + row?.name !== definition.name || + row.origin !== definition.origin || + row.partial !== definition.partial || + row.unique_value !== definition.unique + ) { + return false; + } + const columns = yield* sql.unsafe<{ + readonly cid: unknown; + readonly coll: unknown; + readonly desc_value: unknown; + readonly key_value: unknown; + readonly name: unknown; + readonly seqno: unknown; + }>( + `SELECT seqno, cid, + CASE WHEN name IS NULL THEN NULL + WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name + ELSE 0 END AS name, + "desc" AS desc_value, + CASE WHEN coll = 'BINARY' THEN coll ELSE NULL END AS coll, + "key" AS key_value + FROM pragma_index_xinfo(${sqliteStringLiteral(definition.name)}) + LIMIT ?`, + [definition.columns.length + 1], + ); + if ( + columns.length !== definition.columns.length || + columns.some((column, columnIndex) => { + const expectedColumn = definition.columns[columnIndex]; + return ( + column.seqno !== columnIndex || + column.cid !== expectedColumn?.cid || + column.name !== expectedColumn.name || + column.desc_value !== expectedColumn.desc || + column.coll !== expectedColumn.coll || + column.key_value !== expectedColumn.key + ); + }) + ) { + return false; + } + } + return true; +}); + +export const exactVectorForeignKeys = Effect.fn('codeGraph.exactVectorForeignKeys')(function* ( + sql: SqlClient.SqlClient, + tableName: string, + expected: readonly ExpectedVectorForeignKey[], +) { + const rows = yield* sql.unsafe<{ + readonly from_column: unknown; + readonly id: unknown; + readonly match_value: unknown; + readonly on_delete: unknown; + readonly on_update: unknown; + readonly seq: unknown; + readonly table_name: unknown; + readonly to_column: unknown; + }>( + `SELECT id, seq, + CASE WHEN typeof("table") = 'text' AND length(CAST("table" AS BLOB)) <= 128 + THEN "table" ELSE NULL END AS table_name, + CASE WHEN typeof("from") = 'text' AND length(CAST("from" AS BLOB)) <= 128 + THEN "from" ELSE NULL END AS from_column, + CASE WHEN typeof("to") = 'text' AND length(CAST("to" AS BLOB)) <= 128 + THEN "to" ELSE NULL END AS to_column, + on_update, on_delete, "match" AS match_value + FROM pragma_foreign_key_list(${sqliteStringLiteral(tableName)}) + LIMIT ?`, + [expected.length + 1], + ); + return ( + rows.length === expected.length && + rows.every((row, index) => { + const definition = expected[index]; + return ( + row.id === definition?.id && + row.seq === definition.seq && + row.table_name === definition.table && + row.from_column === definition.from && + row.to_column === definition.to && + row.on_update === definition.onUpdate && + row.on_delete === definition.onDelete && + row.match_value === definition.match + ); + }) + ); +}); + +export const codeGraphVectorCoreSchemaState = Effect.fn('codeGraph.vectorCoreSchemaState')(function* ( + sql: SqlClient.SqlClient, +) { + const expected = [ + {name: 'vector_generations', sql: CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL, type: 'table'}, + {name: 'vector_pointers', sql: CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL, type: 'table'}, + {name: 'vectors', sql: CODE_GRAPH_VECTORS_TABLE_SQL, type: 'table'}, + {name: 'vector_reuse_lookup', sql: CODE_GRAPH_VECTOR_REUSE_INDEX_SQL, type: 'index'}, + ] as const; + for (const object of expected) { + const rows = yield* boundedSchemaObjects(sql, object.name, 2); + if ( + rows.length !== 1 || + rows[0]?.name !== object.name || + rows[0]?.type !== object.type || + normalizeSchemaDefinition(String(rows[0]?.sql ?? '')) !== normalizeSchemaDefinition(object.sql) + ) { + return 'incompatible' as const; + } + } + const userTables = yield* boundedVectorUserTableNames(sql); + const allowedTables = new Set([...VECTOR_CORE_TABLE_NAMES, ...VECTOR_RETIREMENT_TABLE_NAMES]); + if ( + userTables === undefined || + VECTOR_CORE_TABLE_NAMES.some(name => !userTables.includes(name)) || + userTables.some(name => !allowedTables.has(name)) + ) { + return 'incompatible' as const; + } + if ( + !(yield* exactVectorIndexSet(sql, 'vector_generations', VECTOR_GENERATION_INDEXES)) || + !(yield* exactVectorIndexSet(sql, 'vectors', VECTOR_ROW_INDEXES)) || + !(yield* exactVectorForeignKeys(sql, 'vector_generations', [])) || + !(yield* exactVectorForeignKeys(sql, 'vector_pointers', VECTOR_POINTER_FOREIGN_KEYS)) || + !(yield* exactVectorForeignKeys(sql, 'vectors', VECTOR_ROW_FOREIGN_KEYS)) + ) { + return 'incompatible' as const; + } + const pointerIndexesReady = yield* exactVectorIndexSet(sql, 'vector_pointers', VECTOR_POINTER_INDEXES); + const pointerIndexesLegacy = yield* exactVectorIndexSet( + sql, + 'vector_pointers', + VECTOR_POINTER_INDEXES_WITHOUT_GENERATION, + ); + if (!pointerIndexesReady && !pointerIndexesLegacy) return 'incompatible' as const; + const pointerIndex = yield* boundedSchemaObjects(sql, 'vector_pointer_generation_lookup', 2); + if (pointerIndex.length === 0) { + return pointerIndexesLegacy ? ('missing-pointer-index' as const) : ('incompatible' as const); + } + if ( + pointerIndex.length !== 1 || + pointerIndex[0]?.name !== 'vector_pointer_generation_lookup' || + pointerIndex[0]?.type !== 'index' || + normalizeSchemaDefinition(String(pointerIndex[0]?.sql ?? '')) !== + normalizeSchemaDefinition(CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL) + ) { + return 'incompatible' as const; + } + return pointerIndexesReady ? ('ready' as const) : ('incompatible' as const); +}); + +export const codeGraphVectorCoreSchemaCurrent = Effect.fn('codeGraph.vectorCoreSchemaCurrent')(function* ( + sql: SqlClient.SqlClient, +) { + return (yield* codeGraphVectorCoreSchemaState(sql)) === 'ready'; +}); + +export const inspectLegacyPointerIndexPlan = Effect.fn('codeGraph.inspectLegacyVectorPointerIndexPlan')(function* ( + sql: SqlClient.SqlClient, +) { + const statement = codeGraphVectorRetirementLegacyPointerProbeStatement(); + const observed = yield* sql.unsafe<{ + readonly generation: unknown; + readonly identity_bytes: unknown; + readonly worktree_id: unknown; + }>(statement.text, statement.parameters); + if (observed.length > CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS) { + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer index exceeds its bounded migration limit.'), + ); + } + let finalFactBytes = 0; + const rows: Array<{readonly generation: string; readonly worktreeId: string}> = []; + for (const row of observed) { + if ( + typeof row.worktree_id !== 'string' || + !/^[0-9a-f]{64}$/.test(row.worktree_id) || + typeof row.generation !== 'string' || + !validBoundedText(row.generation, VECTOR_GENERATION_BYTES) || + !Number.isSafeInteger(row.identity_bytes) || + Number(row.identity_bytes) <= 64 + ) { + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer index manifest is invalid.'), + ); + } + finalFactBytes += Number(row.identity_bytes); + if (!Number.isSafeInteger(finalFactBytes) || finalFactBytes > CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES) { + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector pointer index exceeds its bounded byte limit.'), + ); + } + rows.push({generation: row.generation, worktreeId: row.worktree_id}); + } + return {finalFactBytes, rows, storage: yield* inspectVectorPageStorageSql(sql)} satisfies LegacyPointerIndexPlan; +}); + +export function sameLegacyPointerIndexPlan(left: LegacyPointerIndexPlan, right: LegacyPointerIndexPlan): boolean { + return ( + left.finalFactBytes === right.finalFactBytes && + sameVectorPageStorage(left.storage, right.storage) && + left.rows.length === right.rows.length && + left.rows.every( + (row, index) => + row.worktreeId === right.rows[index]?.worktreeId && row.generation === right.rows[index]?.generation, + ) + ); +} + +export const codeGraphVectorRetirementSchemaState = Effect.fn('codeGraph.vectorRetirementSchemaState')(function* ( + sql: SqlClient.SqlClient, +) { + const expected = [ + {name: 'vector_retirement_state', sql: CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, type: 'table'}, + {name: 'vector_generation_retirements', sql: CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, type: 'table'}, + { + name: 'vector_generation_retirement_association', + sql: CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, + type: 'index', + }, + ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map(trigger => ({ + name: trigger.name, + sql: trigger.sql, + type: 'trigger' as const, + })), + ]; + const observed: Array<'absent' | 'current' | 'incompatible'> = []; + for (const object of expected) { + const rows = yield* boundedSchemaObjects(sql, object.name, 2); + if (rows.length === 0) { + observed.push('absent'); + continue; + } + observed.push( + rows.length === 1 && + rows[0]?.name === object.name && + rows[0]?.type === object.type && + normalizeSchemaDefinition(String(rows[0]?.sql ?? '')) === normalizeSchemaDefinition(object.sql) + ? 'current' + : 'incompatible', + ); + } + const triggerRows = yield* sql.unsafe<{readonly name: unknown; readonly tbl_name: unknown}>( + `SELECT + CASE WHEN typeof(name) = 'text' AND length(CAST(name AS BLOB)) <= 128 THEN name ELSE NULL END AS name, + CASE WHEN typeof(tbl_name) = 'text' AND length(CAST(tbl_name AS BLOB)) <= 64 THEN tbl_name ELSE NULL END AS tbl_name + FROM sqlite_master + WHERE type = 'trigger' + AND tbl_name COLLATE NOCASE IN ( + 'vector_retirement_state', + 'vector_generation_retirements', + 'vector_generations', + 'vector_pointers', + 'vectors' + ) + LIMIT ?`, + [CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.length + 1], + ); + if (observed.every(state => state === 'absent')) { + const userTables = yield* boundedVectorUserTableNames(sql); + const sequenceTable = yield* boundedSchemaObjects(sql, 'sqlite_sequence', 2); + return triggerRows.length === 0 && sequenceTable.length === 0 && sameStringSet(userTables, VECTOR_CORE_TABLE_NAMES) + ? ('absent' as const) + : ('incompatible' as const); + } + if (!observed.every(state => state === 'current')) return 'incompatible' as const; + const userTables = yield* boundedVectorUserTableNames(sql); + if ( + !sameStringSet(userTables, [...VECTOR_CORE_TABLE_NAMES, ...VECTOR_RETIREMENT_TABLE_NAMES]) || + !(yield* exactVectorIndexSet(sql, 'vector_retirement_state', VECTOR_RETIREMENT_STATE_INDEXES)) || + !(yield* exactVectorIndexSet(sql, 'vector_generation_retirements', VECTOR_RETIREMENT_MARKER_INDEXES)) || + !(yield* exactVectorForeignKeys(sql, 'vector_retirement_state', [])) || + !(yield* exactVectorForeignKeys(sql, 'vector_generation_retirements', [])) + ) { + return 'incompatible' as const; + } + const expectedTriggerNames = [ + ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map( + trigger => `${trigger.name}\0${vectorRetirementTriggerTarget(trigger.name)}`, + ), + ].sort(); + const observedTriggerNames = triggerRows + .map(row => + typeof row.name === 'string' && typeof row.tbl_name === 'string' ? `${row.name}\0${row.tbl_name}` : '', + ) + .sort(); + if ( + observedTriggerNames.length !== expectedTriggerNames.length || + observedTriggerNames.some((name, index) => name !== expectedTriggerNames[index]) + ) { + return 'incompatible' as const; + } + const stateRows = yield* sql.unsafe<{ + readonly admission_cursor: unknown; + readonly admission_scan_revision: unknown; + readonly clean_generation_revision: unknown; + readonly generation_revision: unknown; + readonly pointer_delete_present: unknown; + readonly singleton: unknown; + }>( + `SELECT singleton, + CASE + WHEN admission_cursor IS NULL THEN NULL + WHEN typeof(admission_cursor) = 'text' + AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(admission_cursor, char(0)) = 0 + THEN admission_cursor ELSE 0 + END AS admission_cursor, + CASE + WHEN typeof(generation_revision) = 'integer' + AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN generation_revision ELSE NULL + END AS generation_revision, + CASE + WHEN admission_scan_revision IS NULL THEN NULL + WHEN typeof(admission_scan_revision) = 'integer' + AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN admission_scan_revision ELSE -1 + END AS admission_scan_revision, + CASE + WHEN clean_generation_revision IS NULL THEN NULL + WHEN typeof(clean_generation_revision) = 'integer' + AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN clean_generation_revision ELSE -1 + END AS clean_generation_revision, + CASE + WHEN pointer_delete_worktree_id IS NULL + AND pointer_delete_generation IS NULL + AND pointer_delete_snapshot_id IS NULL + THEN 0 ELSE 1 + END AS pointer_delete_present + FROM vector_retirement_state LIMIT 2`, + ); + if ( + stateRows.length !== 1 || + stateRows[0]?.singleton !== 1 || + !Number.isSafeInteger(stateRows[0]?.generation_revision) || + Number(stateRows[0]?.generation_revision) < 0 || + (stateRows[0]?.admission_scan_revision !== null && + (!Number.isSafeInteger(stateRows[0]?.admission_scan_revision) || + Number(stateRows[0]?.admission_scan_revision) < 0 || + Number(stateRows[0]?.admission_scan_revision) > Number(stateRows[0]?.generation_revision))) || + (stateRows[0]?.clean_generation_revision !== null && + (!Number.isSafeInteger(stateRows[0]?.clean_generation_revision) || + Number(stateRows[0]?.clean_generation_revision) < 0 || + Number(stateRows[0]?.clean_generation_revision) > Number(stateRows[0]?.generation_revision))) || + (stateRows[0]?.admission_cursor === null) !== (stateRows[0]?.admission_scan_revision === null) || + (stateRows[0]?.admission_scan_revision !== null && + stateRows[0]?.clean_generation_revision !== null && + Number(stateRows[0]?.clean_generation_revision) > Number(stateRows[0]?.admission_scan_revision)) || + (stateRows[0]?.clean_generation_revision !== null && + Number(stateRows[0]?.clean_generation_revision) === Number(stateRows[0]?.generation_revision) && + (stateRows[0]?.admission_cursor !== null || stateRows[0]?.admission_scan_revision !== null)) || + stateRows[0]?.pointer_delete_present !== 0 || + (stateRows[0]?.admission_cursor !== null && + (typeof stateRows[0]?.admission_cursor !== 'string' || + !validBoundedText(stateRows[0].admission_cursor, VECTOR_GENERATION_BYTES))) + ) { + return 'incompatible' as const; + } + const sequenceRows = yield* sql.unsafe<{ + readonly name: unknown; + readonly seq: unknown; + readonly seq_type: unknown; + }>( + `SELECT + CASE WHEN typeof(name) = 'text' + AND name = 'vector_generation_retirements' + THEN name ELSE NULL END AS name, + CASE + WHEN typeof(seq) = 'integer' AND seq BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN seq ELSE NULL + END AS seq, + typeof(seq) AS seq_type + FROM sqlite_sequence + WHERE name = 'vector_generation_retirements' COLLATE NOCASE + LIMIT 2`, + ); + const maximumRows = yield* sql.unsafe<{readonly maximum: unknown}>( + `SELECT CASE + WHEN typeof(retirement_id) = 'integer' + AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN retirement_id ELSE NULL + END AS maximum + FROM vector_generation_retirements + ORDER BY retirement_id DESC + LIMIT 1`, + ); + const maximum = maximumRows.length === 0 ? null : maximumRows[0]?.maximum; + if ( + sequenceRows.length !== 1 || + sequenceRows[0]?.name !== 'vector_generation_retirements' || + sequenceRows[0]?.seq_type !== 'integer' || + !Number.isSafeInteger(sequenceRows[0]?.seq) || + Number(sequenceRows[0]?.seq) < 0 || + (maximum !== null && + (!Number.isSafeInteger(maximum) || Number(maximum) <= 0 || Number(sequenceRows[0]?.seq) < Number(maximum))) + ) { + return 'incompatible' as const; + } + return 'ready' as const; +}); + +export const selectVectorRetirementMarker = Effect.fn('codeGraph.selectVectorRetirementMarker')(function* ( + sql: SqlClient.SqlClient, + generation: string, +) { + const rows = yield* sql.unsafe<{ + readonly delete_authorized: unknown; + readonly generation: unknown; + readonly page_revision: unknown; + readonly retired_by_worktree_id: unknown; + readonly retirement_id: unknown; + readonly snapshot_id: unknown; + }>( + `SELECT + CASE + WHEN typeof(retirement_id) = 'integer' + AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN retirement_id ELSE NULL + END AS retirement_id, + CASE + WHEN typeof(generation) = 'text' + AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(generation, char(0)) = 0 + THEN generation ELSE NULL + END AS generation, + CASE + WHEN typeof(snapshot_id) = 'text' + AND length(CAST(snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} + AND instr(snapshot_id, char(0)) = 0 + THEN snapshot_id ELSE NULL + END AS snapshot_id, + CASE + WHEN retired_by_worktree_id IS NULL THEN NULL + WHEN typeof(retired_by_worktree_id) = 'text' + AND length(CAST(retired_by_worktree_id AS BLOB)) = 64 + AND retired_by_worktree_id NOT GLOB '*[^0-9a-f]*' + THEN retired_by_worktree_id ELSE 0 + END AS retired_by_worktree_id, + CASE + WHEN typeof(page_revision) = 'integer' + AND page_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + THEN page_revision ELSE NULL + END AS page_revision, + CASE + WHEN typeof(delete_authorized) = 'integer' AND delete_authorized IN (0, 1) + THEN delete_authorized ELSE NULL + END AS delete_authorized + FROM vector_generation_retirements + WHERE generation = ? LIMIT 2`, + [generation], + ); + if (rows.length === 0) return undefined; + const row = rows[0]; + if ( + rows.length !== 1 || + !Number.isSafeInteger(row?.retirement_id) || + Number(row?.retirement_id) <= 0 || + typeof row?.generation !== 'string' || + !validBoundedText(row.generation, VECTOR_GENERATION_BYTES) || + typeof row?.snapshot_id !== 'string' || + !validBoundedText(row.snapshot_id, VECTOR_SNAPSHOT_BYTES) || + (row?.retired_by_worktree_id !== null && + (typeof row?.retired_by_worktree_id !== 'string' || !/^[0-9a-f]{64}$/.test(row.retired_by_worktree_id))) || + !Number.isSafeInteger(row?.page_revision) || + Number(row?.page_revision) < 0 || + (row?.delete_authorized !== 0 && row?.delete_authorized !== 1) + ) { + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector retirement marker is invalid.')); + } + return { + deleteAuthorized: row.delete_authorized === 1, + generation: row.generation, + pageRevision: Number(row.page_revision), + ...(typeof row.retired_by_worktree_id === 'string' ? {retiredByWorktreeId: row.retired_by_worktree_id} : {}), + retirementId: Number(row.retirement_id), + snapshotId: row.snapshot_id, + } satisfies CodeGraphVectorRetirementMarker; +}); + +export const selectCodeGraphVectorRetirementMarker = selectVectorRetirementMarker; + +export function useExistingVectorDatabase( + databasePath: string, + effect: Effect.Effect, +): Effect.Effect> { + return Effect.scoped( + Layer.build( + SqliteClient.layer({ + create: false, + disableWAL: true, + filename: databasePath, + readwrite: true, + }), + ).pipe(Effect.flatMap(context => effect.pipe(Effect.provide(context)))), + ); +} + +export function vectorRetirementTriggerTarget( + name: (typeof CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS)[number]['name'], +): string { + if (name.includes('_marker_')) return 'vector_generation_retirements'; + if (name.includes('_pointer_')) return 'vector_pointers'; + if (name.includes('_generation_')) return 'vector_generations'; + return 'vectors'; +} + +export function useReadOnlyVectorDatabase( + databasePath: string, + effect: Effect.Effect, +): Effect.Effect> { + return Effect.scoped( + Layer.build( + SqliteClient.layer({ + create: false, + disableWAL: true, + filename: databasePath, + readonly: true, + readwrite: false, + }), + ).pipe(Effect.flatMap(context => effect.pipe(Effect.provide(context)))), + ); +} + +/** @internal Read-only frozen pager tuple for a separately protected cursor publication. */ +export const inspectCodeGraphVectorPageStorage = Effect.fn('codeGraph.inspectVectorPageStorage')(function* ( + databasePath: string, +) { + return yield* useReadOnlyVectorDatabase( + databasePath, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + return yield* inspectVectorPageStorageSql(sql); + }), + ); +}); + +export const inspectVectorPageStorageSql = Effect.fn('codeGraph.inspectVectorPageStorageSql')(function* ( + sql: SqlClient.SqlClient, +) { + const [pageSizeRows, freelistRows, walRows, journalRows] = yield* Effect.all( + [ + sql.unsafe<{readonly page_size: unknown}>('PRAGMA page_size'), + sql.unsafe<{readonly freelist_count: unknown}>('PRAGMA freelist_count'), + sql.unsafe<{readonly wal_autocheckpoint: unknown}>('PRAGMA wal_autocheckpoint'), + sql.unsafe<{readonly journal_mode: unknown}>('PRAGMA journal_mode'), + ] as const, + {concurrency: 1}, + ); + const pageSize = pageSizeRows[0]?.page_size; + const freelistPages = freelistRows[0]?.freelist_count; + const walAutoCheckpointPages = walRows[0]?.wal_autocheckpoint; + const journalMode = journalRows[0]?.journal_mode; + if ( + !Number.isSafeInteger(pageSize) || + Number(pageSize) <= 0 || + !Number.isSafeInteger(freelistPages) || + Number(freelistPages) < 0 || + !Number.isSafeInteger(walAutoCheckpointPages) || + Number(walAutoCheckpointPages) <= 0 || + (journalMode !== 'delete' && journalMode !== 'wal') + ) { + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector page storage is invalid.')); + } + const freelistBytes = Number(pageSize) * Number(freelistPages); + if (!Number.isSafeInteger(freelistBytes)) { + return yield* Effect.fail(new CodeGraphVectorRetirementError('Code graph vector page storage is invalid.')); + } + return { + freelistBytes, + journalMode, + pageSize: Number(pageSize), + walAutoCheckpointPages: Number(walAutoCheckpointPages), + } as const satisfies CodeGraphVectorPageStorage; +}); + +export function sameVectorPageStorage(left: CodeGraphVectorPageStorage, right: CodeGraphVectorPageStorage): boolean { + return ( + left.freelistBytes === right.freelistBytes && + left.pageSize === right.pageSize && + left.walAutoCheckpointPages === right.walAutoCheckpointPages && + left.journalMode === right.journalMode + ); +} + +export function vectorRetirementPageAuthorityBytes(marker: CodeGraphVectorRetirementMarker): number { + return ( + new TextEncoder().encode(marker.generation).byteLength + + new TextEncoder().encode(marker.snapshotId).byteLength + + (marker.retiredByWorktreeId === undefined ? 0 : 64) + + 256 + ); +} + +export function boundedRetirementLimit(requestedLimit: number): number { + if (!Number.isSafeInteger(requestedLimit) || requestedLimit <= 0) { + throw new CodeGraphVectorRetirementError('Code graph vector retirement page limit is invalid.'); + } + return Math.min(requestedLimit, CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS); +} + +export function validBoundedText(value: string, maximumBytes: number): boolean { + return ( + typeof value === 'string' && + value.length > 0 && + !value.includes('\0') && + new TextEncoder().encode(value).byteLength <= maximumBytes + ); +} + +export const boundedSchemaObjects = Effect.fn('codeGraph.boundedVectorSchemaObjects')(function* ( + sql: SqlClient.SqlClient, + name: string, + limit: number, +) { + return yield* sql.unsafe<{ + readonly name: unknown; + readonly sql: unknown; + readonly type: unknown; + }>( + `SELECT name, type, + CASE WHEN typeof(sql) = 'text' AND length(CAST(sql AS BLOB)) <= ? THEN sql ELSE NULL END AS sql + FROM sqlite_master + WHERE name = ? COLLATE NOCASE + LIMIT ?`, + [VECTOR_RETIREMENT_TRIGGER_SQL_BYTES, name, limit], + ); +}); + +export function normalizeSchemaDefinition(value: string): string { + const quoted: string[] = []; + let unquoted = ''; + for (let index = 0; index < value.length; index += 1) { + const opener = value[index]!; + const closer = opener === '[' ? ']' : opener; + if (opener !== "'" && opener !== '"' && opener !== '`' && opener !== '[') { + unquoted += opener; + continue; + } + const start = index; + for (index += 1; index < value.length; index += 1) { + if (value[index] !== closer) continue; + if (closer !== ']' && value[index + 1] === closer) { + index += 1; + continue; + } + break; + } + quoted.push(value.slice(start, Math.min(index + 1, value.length))); + unquoted += `\u0000${quoted.length - 1}\u0000`; + } + return unquoted + .toLowerCase() + .replace(/\bif not exists\b/gu, '') + .replace(/\s+/gu, ' ') + .replace(/\s*([(),])\s*/gu, '$1') + .trim() + .split('\u0000') + .map((segment, index) => (index % 2 === 1 ? (quoted[Number(segment)] ?? '') : segment)) + .join(''); +} + +export const lastStatementChangeCount = Effect.fn('codeGraph.vectorRetirementChangeCount')(function* ( + sql: SqlClient.SqlClient, +) { + const rows = yield* sql.unsafe<{readonly count: unknown}>('SELECT changes() AS count'); + const count = rows[0]?.count; + if (!Number.isSafeInteger(count) || Number(count) < 0) { + return yield* Effect.fail( + new CodeGraphVectorRetirementError('Code graph vector retirement change count is invalid.'), + ); + } + return Number(count); +}); diff --git a/src/code_graph/vector_retirement_schema.ts b/src/code_graph/vector_retirement_schema.ts new file mode 100644 index 00000000..4c0560e3 --- /dev/null +++ b/src/code_graph/vector_retirement_schema.ts @@ -0,0 +1,642 @@ +export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_ROWS = 1_000; +export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_BYTES = 32 * 1_024 * 1_024; +export const CODE_GRAPH_VECTOR_RETIREMENT_PAGE_FIXED_ROWS = 5; +export const CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_ROWS = 8_192; +export const CODE_GRAPH_VECTOR_RETIREMENT_LEGACY_POINTER_BYTES = 4 * 1_024 * 1_024; + +export const MAXIMUM_SAFE_INTEGER_SQL = '9007199254740991'; +export const VECTOR_GENERATION_BYTES = 256; +export const VECTOR_SNAPSHOT_BYTES = 1_024; +export const VECTOR_MODEL_ID_BYTES = 256; +export const VECTOR_MODEL_SHA256_BYTES = 64; +export const VECTOR_CREATED_AT_BYTES = 64; +export const VECTOR_SYMBOL_BYTES = 1_024; +export const VECTOR_FINGERPRINT_BYTES = 1_024; +export const VECTOR_RETIREMENT_TRIGGER_SQL_BYTES = 65_536; +export const VECTOR_CORE_TABLE_NAMES = ['vector_generations', 'vector_pointers', 'vectors'] as const; +export const VECTOR_RETIREMENT_TABLE_NAMES = ['vector_retirement_state', 'vector_generation_retirements'] as const; + +export function storedSchemaSql(value: string): string { + return value.replace(/^CREATE (TABLE|INDEX|TRIGGER) IF NOT EXISTS/u, 'CREATE $1'); +} + +export function sqliteStringLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +export class CodeGraphVectorRetirementError extends Error { + readonly _tag = 'CodeGraphVectorRetirementError' as const; +} + +export function vectorGenerationManifestPredicate(alias: string): string { + return `typeof(${alias}.generation) = 'text' + AND length(CAST(${alias}.generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(${alias}.generation, char(0)) = 0 + AND typeof(${alias}.snapshot_id) = 'text' + AND length(CAST(${alias}.snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} + AND instr(${alias}.snapshot_id, char(0)) = 0 + AND typeof(${alias}.model_id) = 'text' + AND length(CAST(${alias}.model_id AS BLOB)) BETWEEN 1 AND ${VECTOR_MODEL_ID_BYTES} + AND instr(${alias}.model_id, char(0)) = 0 + AND typeof(${alias}.model_sha256) = 'text' + AND length(CAST(${alias}.model_sha256 AS BLOB)) = ${VECTOR_MODEL_SHA256_BYTES} + AND ${alias}.model_sha256 NOT GLOB '*[^0-9a-f]*' + AND typeof(${alias}.dimensions) = 'integer' + AND ${alias}.dimensions BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND typeof(${alias}.template_version) = 'integer' + AND ${alias}.template_version BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND typeof(${alias}.count) = 'integer' + AND ${alias}.count BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND typeof(${alias}.state) = 'text' + AND ${alias}.state IN ('building', 'ready') + AND typeof(${alias}.created_at) = 'text' + AND length(CAST(${alias}.created_at AS BLOB)) BETWEEN 1 AND ${VECTOR_CREATED_AT_BYTES} + AND instr(${alias}.created_at, char(0)) = 0`; +} + +export const CODE_GRAPH_VECTOR_GENERATIONS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_generations ( + generation TEXT PRIMARY KEY, + snapshot_id TEXT NOT NULL, + model_id TEXT NOT NULL, + model_sha256 TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + template_version INTEGER NOT NULL, + count INTEGER NOT NULL CHECK(count >= 0), + state TEXT NOT NULL CHECK(state IN ('building', 'ready')), + created_at TEXT NOT NULL +)`; + +export const CODE_GRAPH_VECTOR_POINTERS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_pointers ( + worktree_id TEXT PRIMARY KEY, + generation TEXT NOT NULL REFERENCES vector_generations(generation) ON DELETE CASCADE +)`; + +export const CODE_GRAPH_VECTOR_POINTER_GENERATION_INDEX_SQL = + 'CREATE INDEX IF NOT EXISTS vector_pointer_generation_lookup ON vector_pointers (generation)'; + +export const CODE_GRAPH_VECTORS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vectors ( + generation TEXT NOT NULL REFERENCES vector_generations(generation) ON DELETE CASCADE, + symbol_id TEXT NOT NULL, + fingerprint TEXT NOT NULL, + vector BLOB NOT NULL, + PRIMARY KEY (generation, symbol_id) +) WITHOUT ROWID`; + +export const CODE_GRAPH_VECTOR_REUSE_INDEX_SQL = + 'CREATE INDEX IF NOT EXISTS vector_reuse_lookup ON vectors (generation, symbol_id, fingerprint)'; + +export const CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_retirement_state ( + singleton INTEGER PRIMARY KEY NOT NULL CHECK ( + typeof(singleton) = 'integer' AND singleton = 1 + ), + admission_cursor TEXT CHECK ( + admission_cursor IS NULL OR ( + typeof(admission_cursor) = 'text' + AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(admission_cursor, char(0)) = 0 + ) + ), + generation_revision INTEGER NOT NULL DEFAULT 0 CHECK ( + typeof(generation_revision) = 'integer' + AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + ), + admission_scan_revision INTEGER CHECK ( + admission_scan_revision IS NULL OR ( + typeof(admission_scan_revision) = 'integer' + AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + ) + ), + clean_generation_revision INTEGER CHECK ( + clean_generation_revision IS NULL OR ( + typeof(clean_generation_revision) = 'integer' + AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + ) + ), + pointer_delete_worktree_id TEXT, + pointer_delete_generation TEXT, + pointer_delete_snapshot_id TEXT, + CHECK ((admission_cursor IS NULL) = (admission_scan_revision IS NULL)), + CHECK (admission_scan_revision IS NULL OR admission_scan_revision <= generation_revision), + CHECK (clean_generation_revision IS NULL OR clean_generation_revision <= generation_revision), + CHECK ( + admission_scan_revision IS NULL + OR clean_generation_revision IS NULL + OR clean_generation_revision <= admission_scan_revision + ), + CHECK ( + clean_generation_revision IS NULL + OR clean_generation_revision < generation_revision + OR (admission_cursor IS NULL AND admission_scan_revision IS NULL) + ), + CHECK ( + ( + pointer_delete_worktree_id IS NULL + AND pointer_delete_generation IS NULL + AND pointer_delete_snapshot_id IS NULL + ) OR ( + typeof(pointer_delete_worktree_id) = 'text' + AND length(CAST(pointer_delete_worktree_id AS BLOB)) = 64 + AND pointer_delete_worktree_id NOT GLOB '*[^0-9a-f]*' + AND typeof(pointer_delete_generation) = 'text' + AND length(CAST(pointer_delete_generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(pointer_delete_generation, char(0)) = 0 + AND typeof(pointer_delete_snapshot_id) = 'text' + AND length(CAST(pointer_delete_snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} + AND instr(pointer_delete_snapshot_id, char(0)) = 0 + ) + ) +) WITHOUT ROWID`; + +export const CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL = `CREATE TABLE IF NOT EXISTS vector_generation_retirements ( + retirement_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK ( + typeof(retirement_id) = 'integer' + AND retirement_id BETWEEN 1 AND ${MAXIMUM_SAFE_INTEGER_SQL} + ), + generation TEXT NOT NULL UNIQUE CHECK ( + typeof(generation) = 'text' + AND length(CAST(generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(generation, char(0)) = 0 + ), + snapshot_id TEXT NOT NULL CHECK ( + typeof(snapshot_id) = 'text' + AND length(CAST(snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} + AND instr(snapshot_id, char(0)) = 0 + ), + retired_by_worktree_id TEXT CHECK ( + retired_by_worktree_id IS NULL OR ( + typeof(retired_by_worktree_id) = 'text' + AND length(CAST(retired_by_worktree_id AS BLOB)) = 64 + AND retired_by_worktree_id NOT GLOB '*[^0-9a-f]*' + ) + ), + page_revision INTEGER NOT NULL DEFAULT 0 CHECK ( + typeof(page_revision) = 'integer' + AND page_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + ), + delete_authorized INTEGER NOT NULL DEFAULT 0 CHECK ( + typeof(delete_authorized) = 'integer' AND delete_authorized IN (0, 1) + ) +)`; + +export const CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL = `CREATE INDEX IF NOT EXISTS vector_generation_retirement_association + ON vector_generation_retirements ( + retired_by_worktree_id, snapshot_id, generation, retirement_id + ) WHERE retired_by_worktree_id IS NOT NULL`; + +export const CORE_SCHEMA_TRIGGER_GUARD_SQL = `SELECT CASE + WHEN NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'index' + AND name = 'vector_pointer_generation_lookup' + AND tbl_name = 'vector_pointers' + LIMIT 1 + ) OR ( + SELECT COUNT(*) FROM ( + SELECT seqno, cid, name, "desc", coll, "key" + FROM pragma_index_xinfo('vector_pointer_generation_lookup') + LIMIT 3 + ) + ) <> 2 OR ( + SELECT COUNT(*) FROM ( + SELECT seqno, cid, name, "desc", coll, "key" + FROM pragma_index_xinfo('vector_pointer_generation_lookup') + LIMIT 3 + ) WHERE ( + seqno = 0 AND name = 'generation' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 + ) OR ( + seqno = 1 AND cid = -1 AND name IS NULL AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 + ) + ) <> 2 OR NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'table' + AND name = 'vectors' + AND tbl_name = 'vectors' + LIMIT 1 + ) OR ( + SELECT COUNT(*) FROM ( + SELECT seqno, cid, name, "desc", coll, "key" + FROM pragma_index_xinfo('sqlite_autoindex_vectors_1') + LIMIT 5 + ) + ) <> 4 OR ( + SELECT COUNT(*) FROM ( + SELECT seqno, cid, name, "desc", coll, "key" + FROM pragma_index_xinfo('sqlite_autoindex_vectors_1') + LIMIT 5 + ) WHERE ( + seqno = 0 AND name = 'generation' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 + ) OR ( + seqno = 1 AND name = 'symbol_id' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 1 + ) OR ( + seqno = 2 AND name = 'fingerprint' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 + ) OR ( + seqno = 3 AND name = 'vector' AND "desc" = 0 AND coll = 'BINARY' AND "key" = 0 + ) + ) <> 4 + THEN RAISE(ABORT, 'code graph vector retirement authority is incompatible') +END;`; + +export const RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL = `${CORE_SCHEMA_TRIGGER_GUARD_SQL} +SELECT CASE + WHEN NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'table' + AND name = 'vector_retirement_state' + AND tbl_name = 'vector_retirement_state' + AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL))} + LIMIT 1 + ) OR NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'table' + AND name = 'vector_generation_retirements' + AND tbl_name = 'vector_generation_retirements' + AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL))} + LIMIT 1 + ) OR NOT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type = 'index' + AND name = 'vector_generation_retirement_association' + AND tbl_name = 'vector_generation_retirements' + AND sql = ${sqliteStringLiteral(storedSchemaSql(CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL))} + LIMIT 1 + ) OR ( + SELECT COUNT(*) FROM ( + SELECT name, typeof(seq) AS seq_type, seq + FROM sqlite_sequence + WHERE name = 'vector_generation_retirements' COLLATE NOCASE + LIMIT 2 + ) + ) <> 1 OR NOT EXISTS ( + SELECT 1 FROM sqlite_sequence + WHERE name = 'vector_generation_retirements' + AND typeof(seq) = 'integer' + AND seq BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND seq >= COALESCE(( + SELECT retirement_id + FROM vector_generation_retirements + ORDER BY retirement_id DESC + LIMIT 1 + ), 0) + LIMIT 1 + ) OR ( + SELECT COUNT(*) FROM (SELECT singleton FROM vector_retirement_state LIMIT 2) + ) <> 1 OR NOT EXISTS ( + SELECT 1 FROM vector_retirement_state + WHERE singleton = 1 + AND typeof(generation_revision) = 'integer' + AND generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND ( + ( + admission_cursor IS NULL + AND admission_scan_revision IS NULL + ) OR ( + typeof(admission_cursor) = 'text' + AND length(CAST(admission_cursor AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(admission_cursor, char(0)) = 0 + AND typeof(admission_scan_revision) = 'integer' + AND admission_scan_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND admission_scan_revision <= generation_revision + ) + ) + AND ( + clean_generation_revision IS NULL OR ( + typeof(clean_generation_revision) = 'integer' + AND clean_generation_revision BETWEEN 0 AND ${MAXIMUM_SAFE_INTEGER_SQL} + AND clean_generation_revision <= generation_revision + ) + ) + AND ( + admission_scan_revision IS NULL + OR clean_generation_revision IS NULL + OR clean_generation_revision <= admission_scan_revision + ) + AND ( + clean_generation_revision IS NULL + OR clean_generation_revision < generation_revision + OR (admission_cursor IS NULL AND admission_scan_revision IS NULL) + ) + AND ( + ( + pointer_delete_worktree_id IS NULL + AND pointer_delete_generation IS NULL + AND pointer_delete_snapshot_id IS NULL + ) OR ( + typeof(pointer_delete_worktree_id) = 'text' + AND length(CAST(pointer_delete_worktree_id AS BLOB)) = 64 + AND pointer_delete_worktree_id NOT GLOB '*[^0-9a-f]*' + AND typeof(pointer_delete_generation) = 'text' + AND length(CAST(pointer_delete_generation AS BLOB)) BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + AND instr(pointer_delete_generation, char(0)) = 0 + AND typeof(pointer_delete_snapshot_id) = 'text' + AND length(CAST(pointer_delete_snapshot_id AS BLOB)) BETWEEN 1 AND ${VECTOR_SNAPSHOT_BYTES} + AND instr(pointer_delete_snapshot_id, char(0)) = 0 + ) + ) + LIMIT 1 + ) + THEN RAISE(ABORT, 'code graph vector retirement marker authority is incompatible') +END;`; + +export const VECTOR_RETIREMENT_MARKER_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_insert_guard + BEFORE INSERT ON vector_generation_retirements + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT CASE + WHEN NEW.retirement_id <> -1 + OR NEW.page_revision <> 0 + OR NEW.delete_authorized <> 0 + OR NOT EXISTS ( + SELECT 1 FROM vector_generations AS generation + WHERE generation.generation = NEW.generation + AND generation.snapshot_id = NEW.snapshot_id + AND ${vectorGenerationManifestPredicate('generation')} + LIMIT 1 + ) + OR EXISTS ( + SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup + WHERE generation = NEW.generation LIMIT 1 + ) + THEN RAISE(ABORT, 'code graph vector retirement marker is invalid') + END; + END`; + +export const VECTOR_RETIREMENT_MARKER_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_update_guard + BEFORE UPDATE ON vector_generation_retirements + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT CASE + WHEN NEW.retirement_id <> OLD.retirement_id + OR NEW.generation <> OLD.generation + OR NEW.snapshot_id <> OLD.snapshot_id + OR NEW.retired_by_worktree_id IS NOT OLD.retired_by_worktree_id + OR NOT EXISTS ( + SELECT 1 FROM vector_generations AS generation + WHERE generation.generation = OLD.generation + AND generation.snapshot_id = OLD.snapshot_id + AND ${vectorGenerationManifestPredicate('generation')} + LIMIT 1 + ) + OR EXISTS ( + SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup + WHERE generation = OLD.generation LIMIT 1 + ) + OR NOT ( + ( + OLD.delete_authorized = 0 + AND NEW.delete_authorized = 0 + AND OLD.page_revision < ${MAXIMUM_SAFE_INTEGER_SQL} + AND NEW.page_revision = OLD.page_revision + 1 + ) OR ( + OLD.delete_authorized = 0 + AND NEW.delete_authorized = 1 + AND NEW.page_revision = OLD.page_revision + AND NOT EXISTS ( + SELECT 1 FROM vectors INDEXED BY sqlite_autoindex_vectors_1 + WHERE generation = OLD.generation LIMIT 1 + ) + ) + ) + THEN RAISE(ABORT, 'code graph vector retirement marker update is invalid') + END; + END`; + +export const VECTOR_RETIREMENT_MARKER_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_marker_delete_guard + BEFORE DELETE ON vector_generation_retirements + WHEN EXISTS ( + SELECT 1 FROM vector_generations WHERE generation = OLD.generation LIMIT 1 + ) + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT RAISE(ABORT, 'code graph vector retirement marker is still authoritative'); + END`; + +export const POINTER_MANIFEST_TRIGGER_GUARD_SQL = `SELECT CASE + WHEN typeof(NEW.worktree_id) <> 'text' + OR length(CAST(NEW.worktree_id AS BLOB)) <> 64 + OR NEW.worktree_id GLOB '*[^0-9a-f]*' + OR typeof(NEW.generation) <> 'text' + OR length(CAST(NEW.generation AS BLOB)) NOT BETWEEN 1 AND ${VECTOR_GENERATION_BYTES} + OR instr(NEW.generation, char(0)) <> 0 + OR NOT EXISTS ( + SELECT 1 FROM vector_generations AS generation + WHERE generation.generation = NEW.generation + AND ${vectorGenerationManifestPredicate('generation')} + LIMIT 1 + ) + THEN RAISE(ABORT, 'code graph vector pointer manifest is invalid') +END;`; + +export const OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL = POINTER_MANIFEST_TRIGGER_GUARD_SQL.replaceAll('NEW.', 'OLD.'); + +export const VECTOR_RETIREMENT_POINTER_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_insert_guard + BEFORE INSERT ON vector_pointers + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + ${POINTER_MANIFEST_TRIGGER_GUARD_SQL} + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements + INDEXED BY sqlite_autoindex_vector_generation_retirements_1 + WHERE generation = NEW.generation LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; + END`; + +export const VECTOR_RETIREMENT_POINTER_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_update_guard + BEFORE UPDATE ON vector_pointers + WHEN NEW.worktree_id <> OLD.worktree_id OR NEW.generation <> OLD.generation + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + ${OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL} + ${POINTER_MANIFEST_TRIGGER_GUARD_SQL} + SELECT CASE WHEN NEW.worktree_id <> OLD.worktree_id + THEN RAISE(ABORT, 'code graph vector pointer identity is immutable') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements + INDEXED BY sqlite_autoindex_vector_generation_retirements_1 + WHERE generation = NEW.generation LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; + END`; + +export const VECTOR_RETIREMENT_POINTER_DELETE_GUARD_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_delete_guard + BEFORE DELETE ON vector_pointers + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + ${OLD_POINTER_MANIFEST_TRIGGER_GUARD_SQL} + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM vector_retirement_state AS authority + JOIN vector_generations AS generation + ON generation.generation = OLD.generation + AND generation.snapshot_id = authority.pointer_delete_snapshot_id + WHERE authority.singleton = 1 + AND authority.pointer_delete_worktree_id = OLD.worktree_id + AND authority.pointer_delete_generation = OLD.generation + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector pointer deletion is unauthorized') END; + END`; + +export const VECTOR_RETIREMENT_POINTER_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_delete_mark + AFTER DELETE ON vector_pointers + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + INSERT INTO vector_generation_retirements ( + generation, snapshot_id, retired_by_worktree_id + ) + SELECT generation, snapshot_id, OLD.worktree_id + FROM vector_generations + WHERE generation = OLD.generation + AND NOT EXISTS ( + SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup + WHERE generation = OLD.generation LIMIT 1 + ); + UPDATE vector_retirement_state + SET pointer_delete_worktree_id = NULL, + pointer_delete_generation = NULL, + pointer_delete_snapshot_id = NULL + WHERE singleton = 1 + AND pointer_delete_worktree_id = OLD.worktree_id + AND pointer_delete_generation = OLD.generation; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM vector_retirement_state + WHERE singleton = 1 AND pointer_delete_worktree_id IS NOT NULL + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector pointer deletion authority was not consumed') END; + END`; + +export const VECTOR_RETIREMENT_POINTER_CHANGED_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_pointer_update_mark + AFTER UPDATE OF generation ON vector_pointers + WHEN NEW.generation <> OLD.generation AND NOT EXISTS ( + SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup + WHERE generation = OLD.generation LIMIT 1 + ) + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + INSERT INTO vector_generation_retirements ( + generation, snapshot_id, retired_by_worktree_id + ) + SELECT generation, snapshot_id, OLD.worktree_id + FROM vector_generations + WHERE generation = OLD.generation; + END`; + +export const VECTOR_RETIREMENT_VECTOR_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_vector_insert_guard + BEFORE INSERT ON vectors + WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements + INDEXED BY sqlite_autoindex_vector_generation_retirements_1 + WHERE generation = NEW.generation LIMIT 1 + ) + BEGIN + SELECT RAISE(ABORT, 'code graph vector generation is retiring'); + END`; + +export const VECTOR_RETIREMENT_VECTOR_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_vector_update_guard + BEFORE UPDATE ON vectors + WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements + INDEXED BY sqlite_autoindex_vector_generation_retirements_1 + WHERE generation = OLD.generation OR generation = NEW.generation + LIMIT 1 + ) + BEGIN + SELECT RAISE(ABORT, 'code graph vector generation is retiring'); + END`; + +export const VECTOR_RETIREMENT_GENERATION_INSERT_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_insert_guard + BEFORE INSERT ON vector_generations + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements WHERE generation = NEW.generation LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM vector_retirement_state + WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; + UPDATE vector_retirement_state + SET generation_revision = generation_revision + 1 + WHERE singleton = 1; + END`; + +export const VECTOR_RETIREMENT_GENERATION_UPDATE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_update_guard + BEFORE UPDATE ON vector_generations + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM vector_generation_retirements + WHERE generation = OLD.generation OR generation = NEW.generation + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation is retiring') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM vector_retirement_state + WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; + UPDATE vector_retirement_state + SET generation_revision = generation_revision + 1 + WHERE singleton = 1; + END`; + +export const VECTOR_RETIREMENT_GENERATION_DELETE_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_delete_guard + BEFORE DELETE ON vector_generations + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + SELECT CASE + WHEN NOT EXISTS ( + SELECT 1 FROM vector_generation_retirements + WHERE generation = OLD.generation AND delete_authorized = 1 + LIMIT 1 + ) OR EXISTS ( + SELECT 1 FROM vector_pointers INDEXED BY vector_pointer_generation_lookup + WHERE generation = OLD.generation LIMIT 1 + ) OR EXISTS ( + SELECT 1 FROM vectors INDEXED BY sqlite_autoindex_vectors_1 + WHERE generation = OLD.generation LIMIT 1 + ) + THEN RAISE(ABORT, 'code graph vector generation deletion is unauthorized') + END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM vector_retirement_state + WHERE singleton = 1 AND generation_revision < ${MAXIMUM_SAFE_INTEGER_SQL} + LIMIT 1 + ) THEN RAISE(ABORT, 'code graph vector generation revision is exhausted') END; + UPDATE vector_retirement_state + SET generation_revision = generation_revision + 1 + WHERE singleton = 1; + END`; + +export const VECTOR_RETIREMENT_GENERATION_DELETED_TRIGGER_SQL = `CREATE TRIGGER IF NOT EXISTS vector_retirement_generation_deleted_clear + AFTER DELETE ON vector_generations + BEGIN + ${RETIREMENT_SCHEMA_TRIGGER_GUARD_SQL} + DELETE FROM vector_generation_retirements + WHERE generation = OLD.generation AND delete_authorized = 1; + END`; + +export const CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS = [ + {name: 'vector_retirement_marker_insert_guard', sql: VECTOR_RETIREMENT_MARKER_INSERT_TRIGGER_SQL}, + {name: 'vector_retirement_marker_update_guard', sql: VECTOR_RETIREMENT_MARKER_UPDATE_TRIGGER_SQL}, + {name: 'vector_retirement_marker_delete_guard', sql: VECTOR_RETIREMENT_MARKER_DELETE_TRIGGER_SQL}, + {name: 'vector_retirement_pointer_insert_guard', sql: VECTOR_RETIREMENT_POINTER_INSERT_TRIGGER_SQL}, + {name: 'vector_retirement_pointer_update_guard', sql: VECTOR_RETIREMENT_POINTER_UPDATE_TRIGGER_SQL}, + {name: 'vector_retirement_pointer_delete_guard', sql: VECTOR_RETIREMENT_POINTER_DELETE_GUARD_TRIGGER_SQL}, + {name: 'vector_retirement_pointer_delete_mark', sql: VECTOR_RETIREMENT_POINTER_DELETE_TRIGGER_SQL}, + {name: 'vector_retirement_pointer_update_mark', sql: VECTOR_RETIREMENT_POINTER_CHANGED_TRIGGER_SQL}, + {name: 'vector_retirement_vector_insert_guard', sql: VECTOR_RETIREMENT_VECTOR_INSERT_TRIGGER_SQL}, + {name: 'vector_retirement_vector_update_guard', sql: VECTOR_RETIREMENT_VECTOR_UPDATE_TRIGGER_SQL}, + {name: 'vector_retirement_generation_insert_guard', sql: VECTOR_RETIREMENT_GENERATION_INSERT_TRIGGER_SQL}, + {name: 'vector_retirement_generation_update_guard', sql: VECTOR_RETIREMENT_GENERATION_UPDATE_TRIGGER_SQL}, + {name: 'vector_retirement_generation_delete_guard', sql: VECTOR_RETIREMENT_GENERATION_DELETE_TRIGGER_SQL}, + {name: 'vector_retirement_generation_deleted_clear', sql: VECTOR_RETIREMENT_GENERATION_DELETED_TRIGGER_SQL}, +] as const; + +// sqlite_schema rows (tables, implicit/explicit indexes, triggers), the +// singleton state row, and sqlite_sequence authority published by r1. The +// fixed conservative count is versioned by the exact SQL byte constant below. +export const CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_ROWS = 24; +export const CODE_GRAPH_VECTOR_RETIREMENT_SCHEMA_FIXED_BYTES = [ + CODE_GRAPH_VECTOR_RETIREMENT_STATE_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENTS_TABLE_SQL, + CODE_GRAPH_VECTOR_RETIREMENT_ASSOCIATION_INDEX_SQL, + ...CODE_GRAPH_VECTOR_RETIREMENT_TRIGGER_DEFINITIONS.map(trigger => trigger.sql), +].reduce((total, sql) => total + new TextEncoder().encode(storedSchemaSql(sql)).byteLength, 256); diff --git a/src/code_graph/view_removal.ts b/src/code_graph/view_removal.ts index dcae0e61..409e2911 100644 --- a/src/code_graph/view_removal.ts +++ b/src/code_graph/view_removal.ts @@ -15,6 +15,10 @@ import { } from './vector_maintenance.js'; import {CODE_GRAPH_SCHEMA_VERSION} from './types.js'; +class CodeGraphViewRemovalError extends Error { + readonly _tag = 'CodeGraphViewRemovalError' as const; +} + const HASH_ID = /^[0-9a-f]{64}$/; const SNAPSHOT_ID = /^cgsn_[0-9a-f]{40}(?:-direct|-full-[0-9a-f]{16})?$/; @@ -111,7 +115,7 @@ export const removeCodeGraphView = Effect.fn('codeGraph.removeViewAction')(funct Effect.flatMap(current => current.state === 'ready' && current.databasePath === inspected.databasePath ? Effect.void - : Effect.fail(new Error('Code graph database target changed before removal.')), + : Effect.fail(new CodeGraphViewRemovalError('Code graph database target changed before removal.')), ), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -191,12 +195,16 @@ export const inspectCodeGraphViewDatabaseTarget = Effect.fn('codeGraph.inspectVi const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; if (Option.isSome(yield* fs.readLink(threadnoteHome).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Threadnote home is a symbolic link; graph view removal was refused.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Threadnote home is a symbolic link; graph view removal was refused.'), + ); } const homeInfo = yield* optionalFileInfo(fs, threadnoteHome); if (Option.isNone(homeInfo)) return {state: 'missing'} as const satisfies CodeGraphViewDatabaseTargetInspection; if (homeInfo.value.type !== 'Directory') { - return yield* Effect.fail(new Error('Threadnote home is not a directory; graph view removal was refused.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Threadnote home is not a directory; graph view removal was refused.'), + ); } const canonicalHome = yield* fs.realPath(threadnoteHome); const segments = [ @@ -210,17 +218,23 @@ export const inspectCodeGraphViewDatabaseTarget = Effect.fn('codeGraph.inspectVi for (const [index, segment] of segments.entries()) { const candidate = path.join(current, segment); if (Option.isSome(yield* fs.readLink(candidate).pipe(Effect.option))) { - return yield* Effect.fail(new Error('Code graph database containment contains a symbolic link.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Code graph database containment contains a symbolic link.'), + ); } const info = yield* optionalFileInfo(fs, candidate); if (Option.isNone(info)) return {state: 'missing'} as const satisfies CodeGraphViewDatabaseTargetInspection; const final = index === segments.length - 1; if ((final && info.value.type !== 'File') || (!final && info.value.type !== 'Directory')) { - return yield* Effect.fail(new Error('Code graph database containment has an invalid entry type.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Code graph database containment has an invalid entry type.'), + ); } const canonical = yield* fs.realPath(candidate); if (canonical !== candidate || path.dirname(canonical) !== current || path.basename(canonical) !== segment) { - return yield* Effect.fail(new Error('Code graph database target escaped its derived-store root.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Code graph database target escaped its derived-store root.'), + ); } current = canonical; } @@ -277,10 +291,10 @@ export function renderCodeGraphViewRemovalResult(result: CodeGraphViewRemovalAct export function codeGraphViewRemovalTargetFailure(result: CodeGraphViewRemovalActionResult): Error | undefined { if (result.state === 'stale-target') { - return new Error('The selected code graph view changed; refresh the view inventory and retry.'); + return new CodeGraphViewRemovalError('The selected code graph view changed; refresh the view inventory and retry.'); } if (result.state === 'not-found') { - return new Error('The selected code graph view does not exist; refresh the view inventory.'); + return new CodeGraphViewRemovalError('The selected code graph view does not exist; refresh the view inventory.'); } return undefined; } @@ -312,12 +326,16 @@ const validateCodeGraphViewRemovalTarget = Effect.fn('codeGraph.validateViewRemo target: CodeGraphViewRemovalTarget, ) { if (!HASH_ID.test(target.checkoutId)) { - return yield* Effect.fail(new Error('Code graph checkout identity must be 64 lowercase hexadecimal characters.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Code graph checkout identity must be 64 lowercase hexadecimal characters.'), + ); } if (!HASH_ID.test(target.worktreeId)) { - return yield* Effect.fail(new Error('Code graph worktree identity must be 64 lowercase hexadecimal characters.')); + return yield* Effect.fail( + new CodeGraphViewRemovalError('Code graph worktree identity must be 64 lowercase hexadecimal characters.'), + ); } if (!SNAPSHOT_ID.test(target.snapshotId)) { - return yield* Effect.fail(new Error('Code graph snapshot identity is invalid.')); + return yield* Effect.fail(new CodeGraphViewRemovalError('Code graph snapshot identity is invalid.')); } }); diff --git a/src/code_graph/visualization.ts b/src/code_graph/visualization.ts index 06d25a64..cddc1e2e 100644 --- a/src/code_graph/visualization.ts +++ b/src/code_graph/visualization.ts @@ -41,6 +41,10 @@ import { export {managerGraphBuildCatalog, type ManagerGraphBuildCatalog} from './manager_status.js'; +class CodeGraphVisualizationError extends Error { + readonly _tag = 'CodeGraphVisualizationError' as const; +} + const NODE_DETAIL_EDGE_LIMIT = 160; const NODE_DETAIL_SUMMARY_LIMIT = 2_000; const MANAGER_CATALOG_PROJECT_LIMIT = 160; @@ -322,7 +326,7 @@ export const managerGraphCatalog = Effect.fn('codeGraph.managerCatalog')(functio diagnostic: { checkoutId, code: 'no-ready-snapshot', - message: `Checkout ${shortIdentity(checkoutId)} has no ready graph snapshot.`, + message: 'An indexed repository database has no ready graph snapshot.', } satisfies ManagerGraphCatalogDiagnostic, } as const; } @@ -347,7 +351,7 @@ export const managerGraphCatalog = Effect.fn('codeGraph.managerCatalog')(functio diagnostic: { checkoutId, code: 'no-ready-snapshot', - message: `Checkout ${shortIdentity(checkoutId)} has no ready graph snapshot.`, + message: 'An indexed repository database has no ready graph snapshot.', } satisfies ManagerGraphCatalogDiagnostic, } as const; } @@ -361,6 +365,14 @@ export const managerGraphCatalog = Effect.fn('codeGraph.managerCatalog')(functio ]).pipe(Effect.map(localAssociation => ({catalog, localAssociation}))), {concurrency: 4}, ); + const primaryView = observedCatalogs[0]; + const primaryViewPath = + primaryView && 'displayPath' in primaryView.localAssociation + ? primaryView.localAssociation.displayPath + : undefined; + const viewLabel = primaryView + ? `${primaryView.catalog.repository.displayName}${primaryViewPath ? ` at ${primaryViewPath}` : ''}` + : 'The indexed repository'; return { checkoutId, catalogs: observedCatalogs, @@ -374,8 +386,8 @@ export const managerGraphCatalog = Effect.fn('codeGraph.managerCatalog')(functio ? ('lease-failed' as const) : ('lease-deferred' as const), message: retained.some(result => result.state === 'failed') - ? `Checkout ${shortIdentity(checkoutId)} remains readable. Snapshot retention is temporarily unavailable, and background maintenance will retry.` - : `Checkout ${shortIdentity(checkoutId)} is readable, but snapshot retention is deferred while another graph writer is active. Retry after the active build completes.`, + ? `${viewLabel} remains readable. Snapshot retention is temporarily unavailable, and background maintenance will retry.` + : `${viewLabel} is readable, but snapshot retention is deferred while another graph writer is active. Retry after the active build completes.`, } satisfies ManagerGraphCatalogDiagnostic, }), viewsTruncated: catalogs.length > MANAGER_CATALOG_VIEW_LIMIT, @@ -388,7 +400,7 @@ export const managerGraphCatalog = Effect.fn('codeGraph.managerCatalog')(functio diagnostic: { checkoutId, code: 'unreadable-database', - message: `Checkout ${shortIdentity(checkoutId)} graph database is unreadable: ${privacySafeCatalogError(cause)}`, + message: `An indexed repository graph database is unreadable: ${privacySafeCatalogError(cause)}`, } satisfies ManagerGraphCatalogDiagnostic, } as const), ), @@ -698,7 +710,9 @@ export const managerGraphCatalogPage = Effect.fn('codeGraph.managerCatalogPage') request: {readonly offset?: number; readonly query?: string; readonly workspaceOffset?: number} = {}, ) { if (Option.isNone(expectedSnapshotId)) { - return yield* Effect.fail(new Error('Graph catalog continuation requires the selected snapshot identity.')); + return yield* Effect.fail( + new CodeGraphVisualizationError('Graph catalog continuation requires the selected snapshot identity.'), + ); } const projectOffset = boundedCatalogOffset(request.offset); const workspaceOffset = boundedCatalogOffset(request.workspaceOffset); @@ -732,13 +746,14 @@ export const managerGraphViewsPage = Effect.fn('codeGraph.managerViewsPage')(fun indexedViewId: string, request: {readonly offset?: number; readonly query?: string} = {}, ) { - if (!INDEXED_VIEW_ID.test(indexedViewId)) return yield* Effect.fail(new Error('Graph view identity is invalid.')); + if (!INDEXED_VIEW_ID.test(indexedViewId)) + return yield* Effect.fail(new CodeGraphVisualizationError('Graph view identity is invalid.')); const path = yield* Path.Path; const store = yield* CodeGraphStore; const [checkoutId] = indexedViewId.split('.', 1) as [string]; const databases = yield* codeGraphDatabasePaths(threadnoteHome); const database = databases.find(candidate => path.basename(path.dirname(candidate)) === checkoutId); - if (!database) return yield* Effect.fail(new Error('Indexed graph checkout was not found.')); + if (!database) return yield* Effect.fail(new CodeGraphVisualizationError('Indexed graph checkout was not found.')); const offset = boundedCatalogOffset(request.offset); const query = boundedCatalogQuery(request.query); const catalogs = yield* store.loadVisualizationCatalogs(database, 'deferred', { @@ -781,7 +796,8 @@ export const managerGraphAnalysis = Effect.fn('codeGraph.managerAnalysis')(funct indexedViewId: string, expectedSnapshotId: Option.Option = Option.none(), ) { - if (!INDEXED_VIEW_ID.test(indexedViewId)) return yield* Effect.fail(new Error('Graph view identity is invalid.')); + if (!INDEXED_VIEW_ID.test(indexedViewId)) + return yield* Effect.fail(new CodeGraphVisualizationError('Graph view identity is invalid.')); const store = yield* CodeGraphStore; return yield* Effect.acquireUseRelease( resolveManagerGraphView(threadnoteHome, indexedViewId, { @@ -811,7 +827,8 @@ export const managerGraphVisualization = Effect.fn('codeGraph.managerVisualizati requestedBudget: ManagerGraphVisualizationBudget = {}, expectedSnapshotId: Option.Option = Option.none(), ) { - if (!INDEXED_VIEW_ID.test(indexedViewId)) return yield* Effect.fail(new Error('Graph view identity is invalid.')); + if (!INDEXED_VIEW_ID.test(indexedViewId)) + return yield* Effect.fail(new CodeGraphVisualizationError('Graph view identity is invalid.')); const store = yield* CodeGraphStore; const projectId = requestedProjectId.trim() || 'all'; return yield* Effect.acquireUseRelease( @@ -829,7 +846,8 @@ export const managerGraphVisualization = Effect.fn('codeGraph.managerVisualizati return yield* overviewVisualization(store, database, repository, catalog, limits); } const project = catalog.projects.find(candidate => candidate.id === projectId); - if (!project) return yield* Effect.fail(new Error('Indexed graph project was not found.')); + if (!project) + return yield* Effect.fail(new CodeGraphVisualizationError('Indexed graph project was not found.')); return yield* detailVisualization(store, database, repository, project, limits); }), resolved => resolved.release, @@ -843,13 +861,16 @@ export const managerGraphQuery = Effect.fn('codeGraph.managerQuery')(function* ( requestedBudget: ManagerGraphVisualizationBudget = {}, expectedSnapshotId: Option.Option = Option.none(), ) { - if (!INDEXED_VIEW_ID.test(indexedViewId)) return yield* Effect.fail(new Error('Graph view identity is invalid.')); + if (!INDEXED_VIEW_ID.test(indexedViewId)) + return yield* Effect.fail(new CodeGraphVisualizationError('Graph view identity is invalid.')); if (Option.isNone(expectedSnapshotId)) { - return yield* Effect.fail(new Error('Graph queries require the selected snapshot identity.')); + return yield* Effect.fail(new CodeGraphVisualizationError('Graph queries require the selected snapshot identity.')); } const query = requestedQuery.trim(); if (query.length === 0 || query.length > MANAGER_QUERY_MAX_LENGTH) { - return yield* Effect.fail(new Error('Graph query must contain between 1 and 512 characters.')); + return yield* Effect.fail( + new CodeGraphVisualizationError('Graph query must contain between 1 and 512 characters.'), + ); } const path = yield* Path.Path; const store = yield* CodeGraphStore; @@ -936,10 +957,11 @@ export const managerGraphNodeDetail = Effect.fn('codeGraph.managerNodeDetail')(f requestedNodeId: string, expectedSnapshotId: Option.Option = Option.none(), ) { - if (!INDEXED_VIEW_ID.test(indexedViewId)) return yield* Effect.fail(new Error('Graph view identity is invalid.')); + if (!INDEXED_VIEW_ID.test(indexedViewId)) + return yield* Effect.fail(new CodeGraphVisualizationError('Graph view identity is invalid.')); const nodeId = requestedNodeId.trim(); if (nodeId.length === 0 || nodeId.length > NODE_ID_MAX_LENGTH) { - return yield* Effect.fail(new Error('Graph node identity is invalid.')); + return yield* Effect.fail(new CodeGraphVisualizationError('Graph node identity is invalid.')); } const store = yield* CodeGraphStore; return yield* Effect.acquireUseRelease( @@ -952,7 +974,7 @@ export const managerGraphNodeDetail = Effect.fn('codeGraph.managerNodeDetail')(f Effect.gen(function* () { const symbols = yield* store.symbolsByIds(database, catalog.snapshot.id, [nodeId]); const symbol = symbols.find(candidate => candidate.id === nodeId); - if (!symbol) return yield* Effect.fail(new Error('Indexed graph node was not found.')); + if (!symbol) return yield* Effect.fail(new CodeGraphVisualizationError('Indexed graph node was not found.')); const [edges, summary] = yield* Effect.all([ store.edgesForNodes( @@ -1422,7 +1444,7 @@ function repositoryFromCatalog( checkoutId, displayName: catalog.repository.displayName, id: viewId, - label: indexedViewLabel(checkoutId, catalog), + label: indexedViewLabel(catalog), localAssociation, metrics: catalog.metrics, model: catalog.model, @@ -1506,7 +1528,7 @@ const resolveManagerGraphView = Effect.fn('codeGraph.resolveManagerGraphView')(f const [checkoutId, worktreeId] = indexedViewId.split('.', 2) as [string, string | undefined]; const databases = yield* codeGraphDatabasePaths(threadnoteHome); const database = databases.find(candidate => path.basename(path.dirname(candidate)) === checkoutId); - if (!database) return yield* Effect.fail(new Error('Indexed graph checkout was not found.')); + if (!database) return yield* Effect.fail(new CodeGraphVisualizationError('Indexed graph checkout was not found.')); const catalogOptions = { includeDependencies: options.includeDependencies, projectOffset: options.projectOffset, @@ -1531,11 +1553,13 @@ const resolveManagerGraphView = Effect.fn('codeGraph.resolveManagerGraphView')(f ? new ManagerGraphViewUnavailableError( 'The selected graph view changed or was removed. Refresh the graph catalog.', ) - : new Error('Indexed graph view has no ready snapshot.'), + : new CodeGraphVisualizationError('Indexed graph view has no ready snapshot.'), ); } if (worktreeId && catalog.viewWorktreeId !== worktreeId) { - return yield* Effect.fail(new Error('Indexed graph snapshot does not belong to the requested view.')); + return yield* Effect.fail( + new CodeGraphVisualizationError('Indexed graph snapshot does not belong to the requested view.'), + ); } const retention = yield* retainManagerSnapshot( store, @@ -1642,16 +1666,12 @@ function compareIndexedViews(left: ManagerGraphIndexedView, right: ManagerGraphI ); } -function indexedViewLabel(checkoutId: string, catalog: CodeGraphVisualizationCatalog): string { +function indexedViewLabel(catalog: CodeGraphVisualizationCatalog): string { const commit = catalog.snapshot.commit.slice(0, 8) || 'no-commit'; const state = catalog.snapshot.dirty ? 'dirty' : 'clean'; const indexed = catalog.activatedAt ?? catalog.snapshot.completedAt; const indexedLabel = indexed ? new Date(indexed).toISOString().slice(0, 16).replace('T', ' ') + 'Z' : 'time unknown'; - return `${commit} · ${state} · ${indexedLabel} · checkout ${shortIdentity(checkoutId)} · worktree ${shortIdentity(catalog.viewWorktreeId)}`; -} - -function shortIdentity(value: string): string { - return value.slice(-8) || 'unknown'; + return `${commit} · ${state} · indexed ${indexedLabel}`; } function privacySafeCatalogError(cause: unknown): string { diff --git a/src/code_graph/watcher.ts b/src/code_graph/watcher.ts index c9824a98..d9e3cd87 100644 --- a/src/code_graph/watcher.ts +++ b/src/code_graph/watcher.ts @@ -104,7 +104,7 @@ export interface CodeGraphWatcherMetrics { export interface CodeGraphWatcherShape { readonly ensure: (options: CodeGraphWatchOptions) => Effect.Effect; - readonly metrics: () => Effect.Effect; + readonly metrics: Effect.Effect; readonly refresh: (options: CodeGraphWatchOptions) => Effect.Effect; readonly status: ( key: string, @@ -133,9 +133,9 @@ export type CodeGraphRecoveryRun = ( ) => Effect.Effect; export interface CodeGraphWatchReconciliationHooks { - readonly periodicRefreshRequired: () => Effect.Effect; - readonly requestAfterChange: () => Effect.Effect; - readonly requestInitial?: () => Effect.Effect; + readonly periodicRefreshRequired: Effect.Effect; + readonly requestAfterChange: Effect.Effect; + readonly requestInitial?: Effect.Effect; } export interface CodeGraphAutomaticRecoveryIdentity extends Partial { @@ -190,6 +190,10 @@ interface RefreshExecutionMetrics { readonly highWater: number; } +class CodeGraphWatcherError extends Error { + readonly _tag = 'CodeGraphWatcherError' as const; +} + const DEFAULT_IDLE_TIMEOUT_MILLISECONDS = 30 * 60_000; const DEFAULT_MAXIMUM_WATCHERS = 32; const DEFAULT_SWEEP_INTERVAL_MILLISECONDS = 60_000; @@ -302,25 +306,23 @@ export class CodeGraphWatcher extends Context.Service ({ - periodicRefreshRequired: () => - Effect.gen(function* () { - const identity = yield* resolveRecoveryIdentity(options.cwd); - yield* requestWatchMaintenance(options, identity); - const layout = codeGraphLayout(path, options.threadnoteHome, identity.checkoutId, identity.worktreeId); - const ready = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); - if (ready === undefined || ready.commit !== identity.headCommit) return true; - const overlay = yield* worktreeOverlayState(identity).pipe( - Effect.provideService(CommandExecutor, commandExecutor), - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(SystemInfo, systemInfo), - ); - return codeGraphWatcherSnapshotStale(ready, identity, overlay); - }), - requestAfterChange: () => - resolveRecoveryIdentity(options.cwd).pipe( - Effect.flatMap(identity => requestWatchMaintenance(options, identity)), - ), + periodicRefreshRequired: Effect.gen(function* () { + const identity = yield* resolveRecoveryIdentity(options.cwd); + yield* requestWatchMaintenance(options, identity); + const layout = codeGraphLayout(path, options.threadnoteHome, identity.checkoutId, identity.worktreeId); + const ready = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); + if (ready === undefined || ready.commit !== identity.headCommit) return true; + const overlay = yield* worktreeOverlayState(identity).pipe( + Effect.provideService(CommandExecutor, commandExecutor), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(SystemInfo, systemInfo), + ); + return codeGraphWatcherSnapshotStale(ready, identity, overlay); + }), + requestAfterChange: resolveRecoveryIdentity(options.cwd).pipe( + Effect.flatMap(identity => requestWatchMaintenance(options, identity)), + ), }); const run = ( options: CodeGraphWatchOptions, @@ -658,28 +660,27 @@ export const makeCodeGraphWatcher = Effect.fn('codeGraph.makeWatcher')(function* return CodeGraphWatcher.of({ ensure: startSessionWatch, - metrics: () => - Effect.gen(function* () { - const watches = yield* SynchronizedRef.get(activeWatches); - const refreshes = yield* SynchronizedRef.get(activeRefreshes); - const statuses = yield* SynchronizedRef.get(refreshStatuses); - const execution = yield* Ref.get(refreshExecutionMetrics); - const idleSweepStarted = yield* Ref.get(sweepStarted); - let pendingTrailingRefreshes = 0; - for (const refresh of refreshes.values()) { - if (refresh.pending) pendingTrailingRefreshes += 1; - } - return { - activeRefreshKeys: refreshes.size, - activeWatches: watches.size, - executingRefreshes: execution.executing, - executingRefreshHighWater: execution.highWater, - idleSweepFibers: idleSweepStarted ? 1 : 0, - maximumWatchers, - pendingTrailingRefreshes, - retainedStatuses: statuses.size, - }; - }), + metrics: Effect.gen(function* () { + const watches = yield* SynchronizedRef.get(activeWatches); + const refreshes = yield* SynchronizedRef.get(activeRefreshes); + const statuses = yield* SynchronizedRef.get(refreshStatuses); + const execution = yield* Ref.get(refreshExecutionMetrics); + const idleSweepStarted = yield* Ref.get(sweepStarted); + let pendingTrailingRefreshes = 0; + for (const refresh of refreshes.values()) { + if (refresh.pending) pendingTrailingRefreshes += 1; + } + return { + activeRefreshKeys: refreshes.size, + activeWatches: watches.size, + executingRefreshes: execution.executing, + executingRefreshHighWater: execution.highWater, + idleSweepFibers: idleSweepStarted ? 1 : 0, + maximumWatchers, + pendingTrailingRefreshes, + retainedStatuses: statuses.size, + }; + }), refresh: options => Effect.gen(function* () { yield* touchWatch(options.key); @@ -717,7 +718,9 @@ export const requestCodeGraphAutomaticRecovery = Effect.fn('codeGraph.requestAut Effect.flatMap(identity => identity.worktreeId === options.key ? dependencies.routineMaintenance(options, identity) - : Effect.fail(new Error('Code graph recovery identity changed before maintenance admission.')), + : Effect.fail( + new CodeGraphWatcherError('Code graph recovery identity changed before maintenance admission.'), + ), ), ); return yield* dependencies.coordinator @@ -983,11 +986,11 @@ export const watchRepository = Effect.fn('codeGraph.watchRepository')(function* _initialRefresh: boolean, requestRefresh: () => Effect.Effect, reconciliationHooks: CodeGraphWatchReconciliationHooks = { - periodicRefreshRequired: () => Effect.succeed(true), - requestAfterChange: () => Effect.void, + periodicRefreshRequired: Effect.succeed(true), + requestAfterChange: Effect.void, }, ) { - yield* (reconciliationHooks.requestInitial ?? reconciliationHooks.requestAfterChange)().pipe( + yield* (reconciliationHooks.requestInitial ?? reconciliationHooks.requestAfterChange).pipe( Effect.catch(() => Effect.logWarning('Code graph initial maintenance scheduling failed; watch remains active.')), ); const changes = fs.watch(options.cwd).pipe( @@ -1006,13 +1009,13 @@ export const watchRepository = Effect.fn('codeGraph.watchRepository')(function* yield* Stream.merge(changes, reconciliation).pipe( Stream.runForEach(event => event === 'change' - ? reconciliationHooks.requestAfterChange().pipe( + ? reconciliationHooks.requestAfterChange.pipe( Effect.catch(() => Effect.logWarning('Code graph change maintenance scheduling failed; refresh remains active.'), ), Effect.andThen(requestRefresh()), ) - : reconciliationHooks.periodicRefreshRequired().pipe( + : reconciliationHooks.periodicRefreshRequired.pipe( Effect.match({ onFailure: () => false, onSuccess: refreshRequired => refreshRequired, diff --git a/src/code_graph/workset_catalog/candidate_source.ts b/src/code_graph/workset_catalog/candidate_source.ts index 2377456d..74a9cb20 100644 --- a/src/code_graph/workset_catalog/candidate_source.ts +++ b/src/code_graph/workset_catalog/candidate_source.ts @@ -1,5 +1,5 @@ import * as SqliteClient from '@effect/sql-sqlite-bun/SqliteClient'; -import {Effect, FileSystem, Path} from 'effect'; +import {Effect, FileSystem, Layer, Path} from 'effect'; import * as SqlClient from 'effect/unstable/sql/SqlClient'; import {sha256HexSync} from '../../crypto/sha256.js'; import type { @@ -143,54 +143,53 @@ function readCandidatePage( if (!(yield* fs.exists(databasePath))) { return yield* Effect.fail(new CodeGraphWorksetCatalogError('missing', 'The workset catalog does not exist.')); } + const read = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* configureCodeGraphWorksetCatalogReadConnection(sql); + return yield* sql.withTransaction( + Effect.gen(function* () { + const memberCount = yield* readCompleteCoverage(sql, normalized); + const coverage = { + consideredMemberCount: memberCount, + eligibleMemberCount: memberCount, + state: 'complete' as const, + }; + if (lane === 'lexical' && normalized.query.terms.length === 0) { + return {coverage, generationId: normalized.generationId, hits: [], lane}; + } + const rows = + lane === 'exact' + ? yield* selectExactCandidates(sql, normalized as typeof normalized & {readonly cursor?: ExactCursor}) + : yield* selectLexicalCandidates( + sql, + normalized as typeof normalized & {readonly cursor?: LexicalCursor}, + ); + const visible = rows.slice(0, normalized.limit); + const surfaces = yield* loadCandidateSurfaces(sql, visible); + const hits = yield* decodeHits(visible, surfaces); + const last = visible.at(-1); + return { + coverage, + generationId: normalized.generationId, + hits, + lane, + ...(rows.length > normalized.limit && last !== undefined + ? {next: encodeCursor(normalized, lane, last)} + : {}), + } satisfies CodeGraphWorksetCatalogCandidatePageV1; + }), + ); + }); return yield* Effect.scoped( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* configureCodeGraphWorksetCatalogReadConnection(sql); - return yield* sql.withTransaction( - Effect.gen(function* () { - const memberCount = yield* readCompleteCoverage(sql, normalized); - const coverage = { - consideredMemberCount: memberCount, - eligibleMemberCount: memberCount, - state: 'complete' as const, - }; - if (lane === 'lexical' && normalized.query.terms.length === 0) { - return {coverage, generationId: normalized.generationId, hits: [], lane}; - } - const rows = - lane === 'exact' - ? yield* selectExactCandidates(sql, normalized as typeof normalized & {readonly cursor?: ExactCursor}) - : yield* selectLexicalCandidates( - sql, - normalized as typeof normalized & {readonly cursor?: LexicalCursor}, - ); - const visible = rows.slice(0, normalized.limit); - const surfaces = yield* loadCandidateSurfaces(sql, visible); - const hits = yield* decodeHits(visible, surfaces); - const last = visible.at(-1); - return { - coverage, - generationId: normalized.generationId, - hits, - lane, - ...(rows.length > normalized.limit && last !== undefined - ? {next: encodeCursor(normalized, lane, last)} - : {}), - } satisfies CodeGraphWorksetCatalogCandidatePageV1; - }), - ); - }).pipe( - Effect.provide( - SqliteClient.layer({ - create: false, - disableWAL: true, - filename: databasePath, - readonly: true, - readwrite: false, - }), - ), - ), + Layer.build( + SqliteClient.layer({ + create: false, + disableWAL: true, + filename: databasePath, + readonly: true, + readwrite: false, + }), + ).pipe(Effect.flatMap(context => read.pipe(Effect.provide(context)))), ); }).pipe(mapCandidateError(`read ${lane} workset candidates`)); } diff --git a/src/code_graph/workset_catalog/layout.ts b/src/code_graph/workset_catalog/layout.ts index 1e2150ac..043b74c7 100644 --- a/src/code_graph/workset_catalog/layout.ts +++ b/src/code_graph/workset_catalog/layout.ts @@ -1,10 +1,11 @@ import type {Path} from 'effect'; -export const CODE_GRAPH_WORKSET_CATALOG_SCHEMA_VERSION = 2 as const; +export const CODE_GRAPH_WORKSET_CATALOG_SCHEMA_VERSION = 3 as const; export interface CodeGraphWorksetCatalogLayout { readonly databasePath: string; readonly lockPath: string; + readonly prepareLockPath: string; readonly root: string; } @@ -32,6 +33,7 @@ export function codeGraphWorksetCatalogLayout(path: Path.Path, threadnoteHome: s return { databasePath: codeGraphWorksetCatalogDatabasePath(path, threadnoteHome), lockPath: codeGraphWorksetCatalogLockPath(path, threadnoteHome), + prepareLockPath: path.join(threadnoteHome, 'locks', 'indexes', 'code-graph', 'worksets', 'prepare.lock'), root: codeGraphWorksetCatalogRoot(path, threadnoteHome), }; } diff --git a/src/code_graph/workset_catalog/projection_builder.ts b/src/code_graph/workset_catalog/projection_builder.ts index b8b4edb7..beb3d5dd 100644 --- a/src/code_graph/workset_catalog/projection_builder.ts +++ b/src/code_graph/workset_catalog/projection_builder.ts @@ -74,11 +74,19 @@ export const stageCodeGraphWorksetRoutingProjectionScoped = Effect.fn( worktreeId: request.identity.worktreeId, }, { - append: (projectionDigest, symbols) => - appendCodeGraphWorksetCatalogProjectionPage(request.threadnoteHome, {projectionDigest, symbols}), - begin: receipt => beginCodeGraphWorksetCatalogProjection(request.threadnoteHome, receipt), - complete: projectionDigest => - completeCodeGraphWorksetCatalogProjection(request.threadnoteHome, projectionDigest).pipe(Effect.asVoid), + append: (projectionDigest, stagingToken, symbols) => + appendCodeGraphWorksetCatalogProjectionPage(request.threadnoteHome, { + projectionDigest, + stagingToken, + symbols, + }), + begin: (receipt, reservedLogicalBytes) => + beginCodeGraphWorksetCatalogProjection(request.threadnoteHome, receipt, reservedLogicalBytes), + complete: (projectionDigest, stagingToken) => + completeCodeGraphWorksetCatalogProjection(request.threadnoteHome, { + projectionDigest, + stagingToken, + }).pipe(Effect.asVoid), }, ); }); diff --git a/src/code_graph/workset_catalog/projection_storage.ts b/src/code_graph/workset_catalog/projection_storage.ts new file mode 100644 index 00000000..4c1fa891 --- /dev/null +++ b/src/code_graph/workset_catalog/projection_storage.ts @@ -0,0 +1,44 @@ +import {CODE_GRAPH_WORKSET_CATALOG_LIMITS, CodeGraphWorksetCatalogError} from './types.js'; +import {codeGraphWorksetRoutingExactKeys} from './routing_normalization.js'; +import type {CodeGraphWorksetRoutingSymbolV1} from './types.js'; + +const ROUTING_ROW_STORAGE_CHARGE_BYTES = 256; + +/** Canonical additive charge for every routing row and derived exact-key row. */ +export function codeGraphWorksetRoutingProjectionLogicalBytes( + symbols: readonly CodeGraphWorksetRoutingSymbolV1[], +): number { + return codeGraphWorksetRoutingProjectionLogicalBytesAppend(0, symbols); +} + +export function codeGraphWorksetRoutingProjectionLogicalBytesAppend( + currentBytes: number, + symbols: readonly CodeGraphWorksetRoutingSymbolV1[], +): number { + if (!Number.isSafeInteger(currentBytes) || currentBytes < 0) { + throw new CodeGraphWorksetCatalogError('invalid-input', 'Workset routing projection byte state is invalid.'); + } + let bytes = currentBytes; + for (const symbol of symbols) { + bytes += Buffer.byteLength(JSON.stringify(symbol), 'utf8') + ROUTING_ROW_STORAGE_CHARGE_BYTES; + for (const lookupKey of symbol.lookupKeys) { + bytes += Buffer.byteLength(lookupKey, 'utf8') + ROUTING_ROW_STORAGE_CHARGE_BYTES; + } + for (const term of symbol.terms) { + bytes += Buffer.byteLength(term.term, 'utf8') + 8 + ROUTING_ROW_STORAGE_CHARGE_BYTES; + } + for (const exactKey of codeGraphWorksetRoutingExactKeys(symbol)) { + bytes += + Buffer.byteLength(exactKey.kind, 'utf8') + + Buffer.byteLength(exactKey.exactKey, 'utf8') + + ROUTING_ROW_STORAGE_CHARGE_BYTES; + } + if (!Number.isSafeInteger(bytes) || bytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionBytesMaximum) { + throw new CodeGraphWorksetCatalogError( + 'capacity', + 'Workset routing projection exceeds the supported aggregate byte bound.', + ); + } + } + return bytes; +} diff --git a/src/code_graph/workset_catalog/schema.ts b/src/code_graph/workset_catalog/schema.ts index 54a8b4a6..b3a5be0f 100644 --- a/src/code_graph/workset_catalog/schema.ts +++ b/src/code_graph/workset_catalog/schema.ts @@ -7,6 +7,10 @@ import { CodeGraphWorksetCatalogError, } from './types.js'; +export const CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES = 4_096; +const CATALOG_PAGE_COUNT_MAXIMUM = + CODE_GRAPH_WORKSET_CATALOG_LIMITS.catalogPhysicalBytesMaximum / CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES; + export const configureCodeGraphWorksetCatalogReadConnection = Effect.fn( 'codeGraphWorksetCatalog.configureReadConnection', )(function* (sql: SqlClient.SqlClient) { @@ -19,15 +23,28 @@ export const configureCodeGraphWorksetCatalogWriteConnection = Effect.fn( )(function* (sql: SqlClient.SqlClient) { yield* sql.unsafe('PRAGMA foreign_keys = ON'); yield* sql.unsafe('PRAGMA busy_timeout = 5000'); + yield* sql.unsafe(`PRAGMA page_size = ${CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES}`); yield* sql.unsafe('PRAGMA journal_mode = WAL'); yield* sql.unsafe('PRAGMA synchronous = FULL'); yield* sql.unsafe('PRAGMA wal_autocheckpoint = 1000'); + yield* sql.unsafe('PRAGMA journal_size_limit = 67108864'); + yield* sql.unsafe(`PRAGMA max_page_count = ${CATALOG_PAGE_COUNT_MAXIMUM}`); }); export const initializeCodeGraphWorksetCatalogSchema = Effect.fn('codeGraphWorksetCatalog.initializeSchema')(function* ( sql: SqlClient.SqlClient, ) { yield* configureCodeGraphWorksetCatalogWriteConnection(sql); + const pageSize = yield* readSqlitePragmaInteger(sql, 'page_size'); + const maximumPages = yield* readSqlitePragmaInteger(sql, 'max_page_count'); + if (pageSize !== CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES || maximumPages !== CATALOG_PAGE_COUNT_MAXIMUM) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError( + 'incompatible', + 'Workset catalog physical capacity settings are incompatible with this release.', + ), + ); + } yield* sql.unsafe(` CREATE TABLE IF NOT EXISTS catalog_metadata ( key TEXT PRIMARY KEY NOT NULL, @@ -66,12 +83,30 @@ export const initializeCodeGraphWorksetCatalogSchema = Effect.fn('codeGraphWorks yield* sql.unsafe(`PRAGMA user_version = ${CODE_GRAPH_WORKSET_CATALOG_SCHEMA_VERSION}`); }); +function readSqlitePragmaInteger(sql: SqlClient.SqlClient, pragma: 'max_page_count' | 'page_size') { + return sql.unsafe>(`PRAGMA ${pragma}`).pipe( + Effect.flatMap(rows => { + const value = rows[0]?.[pragma]; + const parsed = typeof value === 'bigint' ? Number(value) : value; + return typeof parsed === 'number' && Number.isSafeInteger(parsed) && parsed > 0 + ? Effect.succeed(parsed) + : Effect.fail(new CodeGraphWorksetCatalogError('corrupt', 'Workset catalog capacity metadata is invalid.')); + }), + ); +} + export const inspectCodeGraphWorksetCatalogSchemaVersion = Effect.fn('codeGraphWorksetCatalog.inspectSchemaVersion')( function* (sql: SqlClient.SqlClient) { return yield* readCodeGraphWorksetCatalogMetadataInteger(sql, 'schema_version'); }, ); +export const inspectCodeGraphWorksetCatalogPageSize = Effect.fn('codeGraphWorksetCatalog.inspectPageSize')(function* ( + sql: SqlClient.SqlClient, +) { + return yield* readSqlitePragmaInteger(sql, 'page_size'); +}); + function createCodeGraphWorksetCatalogTables(sql: SqlClient.SqlClient) { return Effect.gen(function* () { yield* sql.unsafe(` @@ -87,11 +122,24 @@ function createCodeGraphWorksetCatalogTables(sql: SqlClient.SqlClient) { projector_version INTEGER NOT NULL CHECK(projector_version > 0), component_count INTEGER NOT NULL CHECK(component_count >= 0), symbol_count INTEGER NOT NULL CHECK(symbol_count >= 0), - state TEXT NOT NULL CHECK(state IN ('staging', 'ready')), + state TEXT NOT NULL CHECK(state IN ('staging', 'ready', 'reclaiming')), created_at TEXT NOT NULL, UNIQUE(checkout_id, worktree_id, snapshot_id, projector_version) ) `); + yield* sql.unsafe(` + CREATE TABLE IF NOT EXISTS catalog_capacity ( + singleton INTEGER PRIMARY KEY NOT NULL CHECK(singleton = 1), + bridge_logical_bytes INTEGER NOT NULL CHECK(bridge_logical_bytes >= 0), + projection_logical_bytes INTEGER NOT NULL + CHECK(projection_logical_bytes >= 0), + CHECK(bridge_logical_bytes + projection_logical_bytes <= ${CODE_GRAPH_WORKSET_CATALOG_LIMITS.catalogPhysicalBytesMaximum}) + ) WITHOUT ROWID + `); + yield* sql.unsafe( + `INSERT OR IGNORE INTO catalog_capacity (singleton, bridge_logical_bytes, projection_logical_bytes) + VALUES (1, 0, 0)`, + ); yield* sql.unsafe(` CREATE TABLE IF NOT EXISTS routing_symbols ( projection_digest TEXT NOT NULL REFERENCES repository_snapshots(projection_digest) ON DELETE CASCADE, @@ -110,6 +158,24 @@ function createCodeGraphWorksetCatalogTables(sql: SqlClient.SqlClient) { PRIMARY KEY (projection_digest, node_id) ) WITHOUT ROWID `); + yield* sql.unsafe(` + CREATE TABLE IF NOT EXISTS routing_projection_storage ( + projection_digest TEXT PRIMARY KEY NOT NULL + REFERENCES repository_snapshots(projection_digest) ON DELETE CASCADE, + logical_bytes INTEGER NOT NULL + CHECK(logical_bytes >= 0 AND logical_bytes <= ${CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionBytesMaximum}), + reserved_bytes INTEGER NOT NULL + CHECK(reserved_bytes >= logical_bytes AND reserved_bytes <= ${CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionBytesMaximum}), + staging_token TEXT CHECK(staging_token IS NULL OR length(staging_token) = 64) + ) WITHOUT ROWID + `); + yield* sql.unsafe(` + CREATE TABLE IF NOT EXISTS routing_projection_retirements ( + projection_digest TEXT PRIMARY KEY NOT NULL + REFERENCES repository_snapshots(projection_digest) ON DELETE CASCADE, + requested_at TEXT NOT NULL + ) WITHOUT ROWID + `); yield* sql.unsafe(` CREATE TABLE IF NOT EXISTS routing_lookup_keys ( projection_digest TEXT NOT NULL, @@ -229,6 +295,8 @@ function createCodeGraphWorksetCatalogTables(sql: SqlClient.SqlClient) { resolver_version INTEGER NOT NULL CHECK(resolver_version > 0), bridge_count INTEGER NOT NULL CHECK(bridge_count >= 0 AND bridge_count <= ${CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgesPerGeneration}), + bridge_bytes INTEGER NOT NULL + CHECK(bridge_bytes >= 0 AND bridge_bytes <= ${CODE_GRAPH_WORKSET_CATALOG_LIMITS.bridgeSetBytesMaximum}), bridge_set_digest TEXT NOT NULL CHECK(length(bridge_set_digest) = 64), coverage_state TEXT NOT NULL CHECK(coverage_state IN ('complete', 'partial', 'failed')), repository_count INTEGER NOT NULL @@ -316,6 +384,11 @@ function createCodeGraphWorksetCatalogTables(sql: SqlClient.SqlClient) { CREATE INDEX IF NOT EXISTS workset_generation_members_projection ON workset_generation_members(projection_digest, generation_id) `); + yield* sql.unsafe('DROP INDEX IF EXISTS workset_generations_state_created'); + yield* sql.unsafe(` + CREATE INDEX IF NOT EXISTS routing_projection_retirements_requested + ON routing_projection_retirements(requested_at, projection_digest) + `); yield* sql.unsafe(` CREATE INDEX IF NOT EXISTS qualified_refs_repository_node ON qualified_refs(repository_id, node_id, ref) diff --git a/src/code_graph/workset_catalog/snapshot_projection.ts b/src/code_graph/workset_catalog/snapshot_projection.ts index c979d5ca..6a2061d6 100644 --- a/src/code_graph/workset_catalog/snapshot_projection.ts +++ b/src/code_graph/workset_catalog/snapshot_projection.ts @@ -12,6 +12,7 @@ import { createCodeGraphWorksetRoutingProjection, normalizeCodeGraphWorksetRoutingSymbol, } from './projection.js'; +import {codeGraphWorksetRoutingProjectionLogicalBytesAppend} from './projection_storage.js'; import { CODE_GRAPH_WORKSET_CATALOG_LIMITS, CODE_GRAPH_WORKSET_CATALOG_PROJECTOR_VERSION, @@ -75,12 +76,14 @@ export interface CodeGraphReadySnapshotRoutingProjectionStreamBuildV1 { export interface CodeGraphReadySnapshotRoutingProjectionSinkV1 { readonly append: ( projectionDigest: string, + stagingToken: string, symbols: readonly CodeGraphWorksetRoutingSymbolV1[], ) => Effect.Effect; readonly begin: ( receipt: CodeGraphWorksetRoutingProjectionReceiptV1, - ) => Effect.Effect<{readonly state: 'ready' | 'staging'}, E, R>; - readonly complete: (projectionDigest: string) => Effect.Effect; + reservedLogicalBytes: number, + ) => Effect.Effect<{readonly state: 'ready'} | {readonly stagingToken: string; readonly state: 'staging'}, E, R>; + readonly complete: (projectionDigest: string, stagingToken: string) => Effect.Effect; } interface SnapshotProjectionRow { @@ -490,13 +493,17 @@ function readProjectionStreamed( safeCount(before.edge_count, 'snapshot edge count'), ); const stats = projectionStats(componentCount, dependencyCount); - const firstPass = yield* scanProjectionSymbolPages( - sql, - selected.id, - baseSnapshotId, - input, - stats, - () => Effect.void, + let reservedLogicalBytes = 0; + const firstPass = yield* scanProjectionSymbolPages(sql, selected.id, baseSnapshotId, input, stats, symbols => + Effect.try({ + try: () => { + reservedLogicalBytes = codeGraphWorksetRoutingProjectionLogicalBytesAppend(reservedLogicalBytes, symbols); + }, + catch: cause => + cause instanceof CodeGraphWorksetCatalogError + ? cause + : corrupt('The routing projection storage charge is invalid.', cause), + }), ); if (firstPass.symbolCount !== expectedSymbolCount || firstPass.symbolCount !== selected.symbolCount) { return yield* Effect.fail( @@ -537,7 +544,7 @@ function readProjectionStreamed( catch: cause => corrupt('The ready snapshot contains an invalid streamed projection surface.', cause), }); const receipt = {...header, projectionDigest} satisfies CodeGraphWorksetRoutingProjectionReceiptV1; - const begun = yield* sink.begin(receipt); + const begun = yield* sink.begin(receipt, reservedLogicalBytes); if (begun.state === 'staging') { const verificationStats = projectionStats(componentCount, dependencyCount); const secondPass = yield* scanProjectionSymbolPages( @@ -546,7 +553,7 @@ function readProjectionStreamed( baseSnapshotId, input, verificationStats, - symbols => sink.append(projectionDigest, symbols), + symbols => appendBoundedProjectionPages(sink, projectionDigest, begun.stagingToken, symbols), ); if ( codeGraphWorksetRoutingProjectionDigestComplete(header, secondPass) !== projectionDigest || @@ -555,7 +562,7 @@ function readProjectionStreamed( return yield* Effect.fail(corrupt('The ready snapshot routing projection changed between streaming passes.')); } yield* validateProjectionSnapshotUnchanged(sql, selected, input, before); - yield* sink.complete(projectionDigest); + yield* sink.complete(projectionDigest, begun.stagingToken); } return {receipt, stats: {...stats}}; }).pipe( @@ -585,6 +592,34 @@ function projectionStats(componentCount: number, dependencyCount: number): Mutab }; } +function appendBoundedProjectionPages( + sink: CodeGraphReadySnapshotRoutingProjectionSinkV1, + projectionDigest: string, + stagingToken: string, + symbols: readonly CodeGraphWorksetRoutingSymbolV1[], +) { + return Effect.gen(function* () { + let page: CodeGraphWorksetRoutingSymbolV1[] = []; + let pageBytes = 0; + for (const symbol of symbols) { + const nextBytes = codeGraphWorksetRoutingProjectionLogicalBytesAppend(pageBytes, [symbol]); + if (page.length > 0 && nextBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionPageBytesMaximum) { + yield* sink.append(projectionDigest, stagingToken, page); + page = []; + pageBytes = 0; + } + pageBytes = codeGraphWorksetRoutingProjectionLogicalBytesAppend(pageBytes, [symbol]); + if (pageBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionPageBytesMaximum) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('capacity', 'A routing symbol exceeds the supported projection page bound.'), + ); + } + page.push(symbol); + } + if (page.length > 0) yield* sink.append(projectionDigest, stagingToken, page); + }); +} + function scanProjectionSymbolPages( sql: SqlClient.SqlClient, snapshotId: string, diff --git a/src/code_graph/workset_catalog/storage_capacity.ts b/src/code_graph/workset_catalog/storage_capacity.ts new file mode 100644 index 00000000..52251bd7 --- /dev/null +++ b/src/code_graph/workset_catalog/storage_capacity.ts @@ -0,0 +1,58 @@ +import {Effect} from 'effect'; +import {CodeGraphWorksetCatalogError} from './types.js'; + +const CATALOG_DISK_SAFETY_BYTES = 512 * 1_024 * 1_024; +const CATALOG_WRITE_AMPLIFICATION = 5; +const CATALOG_ROW_STORAGE_OVERHEAD_BYTES = 256; + +export function codeGraphWorksetCatalogWriteRequiredFreeBytes(payloadBytes: number, rows: number): number { + if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 0 || !Number.isSafeInteger(rows) || rows < 0) { + throw invalidCapacityInput(); + } + const safetyBytes = Math.max(CATALOG_DISK_SAFETY_BYTES, Math.ceil(payloadBytes * 0.1)); + const requiredBytes = + payloadBytes * CATALOG_WRITE_AMPLIFICATION + rows * CATALOG_ROW_STORAGE_OVERHEAD_BYTES + safetyBytes; + if (!Number.isSafeInteger(requiredBytes)) throw invalidCapacityInput(); + return requiredBytes; +} + +export function verifyCodeGraphWorksetCatalogDiskCapacity( + probe: (target: string) => Effect.Effect, + target: string, + requiredBytes: number, + operation: string, +) { + return probe(target).pipe( + Effect.mapError( + cause => + new CodeGraphWorksetCatalogError( + 'storage', + `Could not inspect free disk space before ${operation}. Verify at least ${String(requiredBytes)} bytes are free and retry; the requested data was not staged.`, + {cause}, + ), + ), + Effect.flatMap(availableBytes => { + if (availableBytes === undefined || !Number.isSafeInteger(availableBytes) || availableBytes < 0) { + return Effect.fail( + new CodeGraphWorksetCatalogError( + 'storage', + `Could not determine free disk space before ${operation}. Verify at least ${String(requiredBytes)} bytes are free and retry; the requested data was not staged.`, + ), + ); + } + if (availableBytes < requiredBytes) { + return Effect.fail( + new CodeGraphWorksetCatalogError( + 'capacity', + `${operation} needs ${String(requiredBytes)} bytes free, but only ${String(availableBytes)} bytes are available. Free disk space and retry; the requested data was not staged.`, + ), + ); + } + return Effect.void; + }), + ); +} + +function invalidCapacityInput() { + return new CodeGraphWorksetCatalogError('invalid-input', 'Workset catalog storage estimate is invalid.'); +} diff --git a/src/code_graph/workset_catalog/store.ts b/src/code_graph/workset_catalog/store.ts index b70af568..68a11a84 100644 --- a/src/code_graph/workset_catalog/store.ts +++ b/src/code_graph/workset_catalog/store.ts @@ -1,6 +1,8 @@ import {Clock, Crypto, Effect, FileSystem, Path} from 'effect'; +import * as SqlClient from 'effect/unstable/sql/SqlClient'; import {sha256HexSync} from '../../crypto/sha256.js'; import {withExclusiveFileLock} from '../../effect/file_lock.js'; +import {SystemInfo} from '../../effect/system.js'; import { CODE_GRAPH_WORKSET_EVIDENCE_PROJECTOR_VERSION, codeGraphQualifiedRefHandle, @@ -49,8 +51,8 @@ import { selectProjectionForSnapshot, selectProjectionByDigest, projectionState, + dropQueuedReferencedProjections, insertProjectionHeader, - stageProjection, insertRoutingSymbol, loadAndValidateProjection, decodeProjectionMetadata, @@ -95,6 +97,14 @@ import { invalid, corrupt, } from './store_support.js'; +import { + codeGraphWorksetRoutingProjectionLogicalBytes, + codeGraphWorksetRoutingProjectionLogicalBytesAppend, +} from './projection_storage.js'; +import { + codeGraphWorksetCatalogWriteRequiredFreeBytes, + verifyCodeGraphWorksetCatalogDiskCapacity, +} from './storage_capacity.js'; export {withCodeGraphWorksetCatalogReader, withCodeGraphWorksetCatalogWriter} from './store_support.js'; const CATALOG_LOCK_OPTIONS = { @@ -109,6 +119,9 @@ const QUALIFIED_REF = /^cgr_[0-9a-f]{40}$/u; const CONTINUATION_CURSOR = /^cgwc_[0-9a-f]{40}$/u; const SHA256_HEX = /^[0-9a-f]{64}$/u; const LOCAL_NODE_ID = /^cgs_(?:[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$/u; +const PRODUCTION_MAINTENANCE_LIMIT = 32; +const STAGING_GENERATION_RETENTION_MILLISECONDS = 24 * 60 * 60 * 1_000; +const CATALOG_RECLAIM_ROW_BUDGET = 256; interface GenerationRow { readonly generation_digest: unknown; @@ -216,8 +229,24 @@ export const ensureCodeGraphWorksetCatalog = Effect.fn('codeGraphWorksetCatalog. export const beginCodeGraphWorksetCatalogProjection = Effect.fn('codeGraphWorksetCatalog.beginProjection')(function* ( threadnoteHome: string, input: CodeGraphWorksetRoutingProjectionReceiptV1, + reservedLogicalBytes: number, ) { - const receipt = yield* validateInput(() => validateCodeGraphWorksetRoutingProjectionReceipt(input)); + const receipt = yield* validateInput(() => { + const validated = validateCodeGraphWorksetRoutingProjectionReceipt(input); + if ( + !Number.isSafeInteger(reservedLogicalBytes) || + reservedLogicalBytes < 0 || + reservedLogicalBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionBytesMaximum + ) { + throw invalid('Workset routing projection reservation is invalid.'); + } + return validated; + }); + const crypto = yield* Crypto.Crypto; + const stagingToken = sha256HexSync(yield* crypto.randomBytes(32)); + const path = yield* Path.Path; + const system = yield* SystemInfo; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); return yield* withCatalogWriter(threadnoteHome, sql => Effect.gen(function* () { const now = yield* currentIsoInstant; @@ -232,15 +261,23 @@ export const beginCodeGraphWorksetCatalogProjection = Effect.fn('codeGraphWorkse yield* loadAndValidateProjection(sql, receipt.projectionDigest, true); return {receipt, state: 'ready' as const}; } - yield* sql.withTransaction( - sql.unsafe('DELETE FROM repository_snapshots WHERE projection_digest = ? AND state = ?', [ - receipt.projectionDigest, - 'staging', - ]), + return yield* Effect.fail( + new CodeGraphWorksetCatalogError( + existing.state === 'reclaiming' ? 'capacity' : 'busy', + existing.state === 'reclaiming' + ? 'Routing projection cleanup must finish before restaging.' + : 'Another routing projection stream is already staging this snapshot.', + ), ); } - yield* insertProjectionHeader(sql, receipt, now); - return {receipt, state: 'staging' as const}; + yield* verifyCodeGraphWorksetCatalogDiskCapacity( + target => system.availableDiskBytes(target), + layout.root, + codeGraphWorksetCatalogWriteRequiredFreeBytes(reservedLogicalBytes, receipt.symbolCount), + 'routing projection reservation', + ); + yield* insertProjectionHeader(sql, receipt, now, reservedLogicalBytes, stagingToken); + return {receipt, stagingToken, state: 'staging' as const}; }), ); }); @@ -249,10 +286,15 @@ export const beginCodeGraphWorksetCatalogProjection = Effect.fn('codeGraphWorkse export const appendCodeGraphWorksetCatalogProjectionPage = Effect.fn('codeGraphWorksetCatalog.appendProjectionPage')( function* ( threadnoteHome: string, - input: {readonly projectionDigest: string; readonly symbols: readonly CodeGraphWorksetRoutingSymbolV1[]}, + input: { + readonly projectionDigest: string; + readonly stagingToken: string; + readonly symbols: readonly CodeGraphWorksetRoutingSymbolV1[]; + }, ) { - yield* validateInput(() => { + const pageBytes = yield* validateInput(() => { if (!SHA256_HEX.test(input.projectionDigest)) throw invalid('Workset projection digest is invalid.'); + if (!SHA256_HEX.test(input.stagingToken)) throw invalid('Workset projection staging token is invalid.'); if (input.symbols.length < 1 || input.symbols.length > CODE_GRAPH_WORKSET_CATALOG_PROJECTION_PAGE_MAXIMUM) { throw invalid('Workset projection page size is invalid.'); } @@ -260,7 +302,24 @@ export const appendCodeGraphWorksetCatalogProjectionPage = Effect.fn('codeGraphW codeGraphWorksetRoutingProjectionDigestStart(), input.symbols, ); + const bytes = codeGraphWorksetRoutingProjectionLogicalBytes(input.symbols); + if (bytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionPageBytesMaximum) { + throw new CodeGraphWorksetCatalogError( + 'capacity', + 'Workset routing projection page exceeds the supported aggregate byte bound.', + ); + } + return bytes; }); + const path = yield* Path.Path; + const system = yield* SystemInfo; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); + yield* verifyCodeGraphWorksetCatalogDiskCapacity( + target => system.availableDiskBytes(target), + layout.root, + codeGraphWorksetCatalogWriteRequiredFreeBytes(pageBytes, input.symbols.length), + 'routing projection page', + ); yield* withCatalogWriter(threadnoteHome, sql => sql.withTransaction( Effect.gen(function* () { @@ -268,6 +327,37 @@ export const appendCodeGraphWorksetCatalogProjectionPage = Effect.fn('codeGraphW if (state !== 'staging') { return yield* Effect.fail(new CodeGraphWorksetCatalogError('stale', 'Projection staging is not active.')); } + yield* sql.unsafe( + `UPDATE routing_projection_storage + SET logical_bytes = logical_bytes + ? + WHERE projection_digest = ? AND staging_token = ? + AND logical_bytes <= reserved_bytes - ?`, + [pageBytes, input.projectionDigest, input.stagingToken, pageBytes], + ); + if ((yield* changes(sql)) !== 1) { + const receipts = yield* sql.unsafe<{ + readonly logical_bytes: unknown; + readonly staging_token: unknown; + }>( + `SELECT logical_bytes, staging_token FROM routing_projection_storage + WHERE projection_digest = ? LIMIT 1`, + [input.projectionDigest], + ); + if (receipts.length === 0) { + return yield* Effect.fail(corrupt('Routing projection storage receipt is missing.')); + } + if (receipts[0]!.staging_token !== input.stagingToken) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('stale', 'Routing projection staging ownership changed.'), + ); + } + return yield* Effect.fail( + new CodeGraphWorksetCatalogError( + 'capacity', + 'Workset routing projection exceeds the supported aggregate byte bound.', + ), + ); + } yield* Effect.forEach(input.symbols, symbol => insertRoutingSymbol(sql, input.projectionDigest, symbol), { concurrency: 1, discard: true, @@ -280,24 +370,76 @@ export const appendCodeGraphWorksetCatalogProjectionPage = Effect.fn('codeGraphW /** Recompute streamed integrity and make one fully appended projection eligible for a generation. */ export const completeCodeGraphWorksetCatalogProjection = Effect.fn('codeGraphWorksetCatalog.completeProjection')( - function* (threadnoteHome: string, projectionDigest: string) { + function* (threadnoteHome: string, input: {readonly projectionDigest: string; readonly stagingToken: string}) { yield* validateInput(() => { - if (!SHA256_HEX.test(projectionDigest)) throw invalid('Workset projection digest is invalid.'); + if (!SHA256_HEX.test(input.projectionDigest)) throw invalid('Workset projection digest is invalid.'); + if (!SHA256_HEX.test(input.stagingToken)) throw invalid('Workset projection staging token is invalid.'); }); return yield* withCatalogWriter(threadnoteHome, sql => Effect.gen(function* () { - const projection = yield* loadAndValidateProjection(sql, projectionDigest); - if (projection.state === 'ready') return projection.receipt; - yield* sql.withTransaction( - sql.unsafe( - `UPDATE repository_snapshots SET state = 'ready' - WHERE projection_digest = ? AND state = 'staging'`, - [projectionDigest], - ), + const ownership = yield* sql.unsafe<{ + readonly staging_token: unknown; + readonly state: unknown; + }>( + `SELECT p.state, s.staging_token + FROM repository_snapshots AS p + JOIN routing_projection_storage AS s USING (projection_digest) + WHERE p.projection_digest = ? LIMIT 1`, + [input.projectionDigest], ); - if ((yield* changes(sql)) !== 1) { - return yield* Effect.fail(corrupt('Routing projection publication lost its staging state.')); + if (ownership.length !== 1) { + return yield* Effect.fail(new CodeGraphWorksetCatalogError('stale', 'Projection staging is not active.')); } + if (ownership[0]!.state === 'ready') { + return (yield* loadAndValidateProjection(sql, input.projectionDigest, true)).receipt; + } + if (ownership[0]!.state !== 'staging' || ownership[0]!.staging_token !== input.stagingToken) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('stale', 'Routing projection staging ownership changed.'), + ); + } + const projection = yield* loadAndValidateProjection(sql, input.projectionDigest); + yield* sql.withTransaction( + Effect.gen(function* () { + const storage = yield* sql.unsafe<{ + readonly logical_bytes: unknown; + readonly reserved_bytes: unknown; + readonly staging_token: unknown; + }>( + `SELECT logical_bytes, reserved_bytes, staging_token FROM routing_projection_storage + WHERE projection_digest = ? AND staging_token = ? + LIMIT 1`, + [input.projectionDigest, input.stagingToken], + ); + if (storage.length !== 1) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('stale', 'Routing projection staging ownership changed.'), + ); + } + if ( + requiredInteger(storage[0]!.logical_bytes, 'routing projection logical bytes') !== + requiredInteger(storage[0]!.reserved_bytes, 'routing projection reserved bytes') + ) { + return yield* Effect.fail(corrupt('Routing projection storage reservation is incomplete.')); + } + yield* sql.unsafe( + `UPDATE repository_snapshots SET state = 'ready' + WHERE projection_digest = ? AND state = 'staging'`, + [input.projectionDigest], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Routing projection publication lost its staging state.')); + } + yield* sql.unsafe( + `UPDATE routing_projection_storage SET staging_token = NULL + WHERE projection_digest = ? AND staging_token = ?`, + [input.projectionDigest, input.stagingToken], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Routing projection staging ownership changed before publication.')); + } + }), + ); return projection.receipt; }), ); @@ -314,71 +456,112 @@ export const stageCodeGraphWorksetCatalogGeneration = Effect.fn('codeGraphWorkse input: CodeGraphWorksetCatalogGenerationInputV1, ) { const identity = yield* validateInput(() => codeGraphWorksetCatalogGenerationIdentity(input)); - return yield* withCatalogWriter(threadnoteHome, sql => - Effect.gen(function* () { - const now = yield* currentIsoInstant; - const existing = yield* selectGeneration(sql, identity.id); - if (existing?.state === 'ready') { - const published = yield* generationIsPublished(sql, identity.id, input.worksetName); - if (!published) { - return yield* Effect.fail(corrupt('A ready workset generation has no matching published pointer.')); - } - return generationReceipt(existing); - } - yield* sql.withTransaction( - Effect.gen(function* () { - if (existing !== undefined) { - yield* sql.unsafe('DELETE FROM workset_generations WHERE id = ?', [identity.id]); - } - yield* sql.unsafe( - `INSERT INTO workset_generations ( - id, workset_name, manifest_digest, generation_digest, state, - member_count, created_at, published_at - ) VALUES (?, ?, ?, ?, 'staging', ?, ?, NULL)`, - [identity.id, input.worksetName, input.manifestDigest, identity.digest, identity.members.length, now], - ); - }), - ); - - for (let ordinal = 0; ordinal < identity.members.length; ordinal += 1) { - const member = identity.members[ordinal]!; - yield* stageProjection(sql, member.projection, now); - yield* sql.withTransaction( - sql.unsafe( - `INSERT INTO workset_generation_members ( - generation_id, ordinal, repository_key, repository_id, snapshot_id, projection_digest - ) VALUES (?, ?, ?, ?, ?, ?)`, - [ - identity.id, - ordinal, - member.repositoryKey, - member.projection.repositoryId, - member.projection.snapshotId, - member.projection.projectionDigest, - ], - ), + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); + const ownedProjectionDigests: string[] = []; + const critical = Effect.gen(function* () { + yield* drainCodeGraphWorksetCatalogPreparationPages(threadnoteHome); + const members: CodeGraphWorksetCatalogGenerationDigestMemberV1[] = []; + for (const member of identity.members) { + const projection = member.projection; + const projectionReceipt = { + checkoutId: projection.checkoutId, + commitId: projection.commitId, + componentCount: projection.componentCount, + extractorGeneration: projection.extractorGeneration, + projectionDigest: projection.projectionDigest, + projectorVersion: projection.projectorVersion, + repositoryId: projection.repositoryId, + snapshotDigest: projection.snapshotDigest, + snapshotId: projection.snapshotId, + symbolCount: projection.symbols.length, + worktreeId: projection.worktreeId, + } satisfies CodeGraphWorksetRoutingProjectionReceiptV1; + const reservation = codeGraphWorksetRoutingProjectionLogicalBytes(projection.symbols); + const staged = yield* beginCodeGraphWorksetCatalogProjection(threadnoteHome, projectionReceipt, reservation); + if (staged.state === 'staging') { + ownedProjectionDigests.push(projection.projectionDigest); + yield* appendFullProjectionPages( + threadnoteHome, + projection.projectionDigest, + staged.stagingToken, + projection.symbols, ); + yield* completeCodeGraphWorksetCatalogProjection(threadnoteHome, { + projectionDigest: projection.projectionDigest, + stagingToken: staged.stagingToken, + }); } - const count = yield* rowCount( - sql, - 'SELECT COUNT(*) AS count FROM workset_generation_members WHERE generation_id = ?', - [identity.id], - ); - if (count !== identity.members.length) { - return yield* Effect.fail(corrupt('Staged workset generation member count is inconsistent.')); - } - return { - digest: identity.digest, - id: identity.id, - manifestDigest: input.manifestDigest, - memberCount: identity.members.length, - state: 'staging' as const, - worksetName: input.worksetName, - }; - }), + members.push({ + projectionDigest: projection.projectionDigest, + repositoryId: projection.repositoryId, + repositoryKey: member.repositoryKey, + snapshotId: projection.snapshotId, + }); + } + return yield* stageCodeGraphWorksetCatalogGenerationFromReceipts(threadnoteHome, { + manifestDigest: input.manifestDigest, + members, + worksetName: input.worksetName, + }); + }).pipe( + Effect.onError(() => + retireCodeGraphWorksetCatalogPreparation(threadnoteHome, { + projectionDigests: ownedProjectionDigests, + }).pipe( + Effect.andThen(drainCodeGraphWorksetCatalogPreparationPages(threadnoteHome)), + Effect.catchCause(() => Effect.void), + ), + ), + ); + return yield* withExclusiveFileLock(fs, layout.prepareLockPath, CATALOG_LOCK_OPTIONS, critical).pipe( + mapCatalogError('serialize workset catalog preparation'), ); }); +function appendFullProjectionPages( + threadnoteHome: string, + projectionDigest: string, + stagingToken: string, + symbols: readonly CodeGraphWorksetRoutingSymbolV1[], +) { + return Effect.gen(function* () { + let page: CodeGraphWorksetRoutingSymbolV1[] = []; + let pageBytes = 0; + for (const symbol of symbols) { + const nextBytes = codeGraphWorksetRoutingProjectionLogicalBytesAppend(pageBytes, [symbol]); + if ( + page.length > 0 && + (page.length === CODE_GRAPH_WORKSET_CATALOG_PROJECTION_PAGE_MAXIMUM || + nextBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionPageBytesMaximum) + ) { + yield* appendCodeGraphWorksetCatalogProjectionPage(threadnoteHome, { + projectionDigest, + stagingToken, + symbols: page, + }); + page = []; + pageBytes = 0; + } + pageBytes = codeGraphWorksetRoutingProjectionLogicalBytesAppend(pageBytes, [symbol]); + if (pageBytes > CODE_GRAPH_WORKSET_CATALOG_LIMITS.projectionPageBytesMaximum) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('capacity', 'A routing symbol exceeds the supported projection page bound.'), + ); + } + page.push(symbol); + } + if (page.length > 0) { + yield* appendCodeGraphWorksetCatalogProjectionPage(threadnoteHome, { + projectionDigest, + stagingToken, + symbols: page, + }); + } + }); +} + /** Stage a deterministic generation from lightweight, already-streamed projection receipts. */ export const stageCodeGraphWorksetCatalogGenerationFromReceipts = Effect.fn( 'codeGraphWorksetCatalog.stageGenerationFromReceipts', @@ -394,6 +577,28 @@ export const stageCodeGraphWorksetCatalogGenerationFromReceipts = Effect.fn( } return generationReceipt(existing); } + if (existing?.state === 'staging') { + const stagedMembers = yield* loadGenerationMembers(sql, identity.id); + const matches = + stagedMembers.length === identity.members.length && + stagedMembers.every((member, ordinal) => { + const expected = identity.members[ordinal]; + return ( + expected !== undefined && + member.repository_key === expected.repositoryKey && + member.repository_id === expected.repositoryId && + member.snapshot_id === expected.snapshotId && + member.projection_digest === expected.projectionDigest + ); + }); + if (!matches) return yield* Effect.fail(corrupt('An existing staging generation is incomplete.')); + return generationReceipt(existing); + } + if (existing?.state === 'retired') { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('capacity', 'Retired catalog cleanup must finish before restaging.'), + ); + } for (const member of identity.members) { const projection = yield* selectProjectionByDigest(sql, member.projectionDigest); if ( @@ -409,7 +614,6 @@ export const stageCodeGraphWorksetCatalogGenerationFromReceipts = Effect.fn( } yield* sql.withTransaction( Effect.gen(function* () { - if (existing !== undefined) yield* sql.unsafe('DELETE FROM workset_generations WHERE id = ?', [identity.id]); yield* sql.unsafe( `INSERT INTO workset_generations ( id, workset_name, manifest_digest, generation_digest, state, @@ -417,27 +621,28 @@ export const stageCodeGraphWorksetCatalogGenerationFromReceipts = Effect.fn( ) VALUES (?, ?, ?, ?, 'staging', ?, ?, NULL)`, [identity.id, input.worksetName, input.manifestDigest, identity.digest, identity.members.length, now], ); + for (let ordinal = 0; ordinal < identity.members.length; ordinal += 1) { + const member = identity.members[ordinal]!; + yield* sql.unsafe( + `INSERT INTO workset_generation_members ( + generation_id, ordinal, repository_key, repository_id, snapshot_id, projection_digest + ) VALUES (?, ?, ?, ?, ?, ?)`, + [ + identity.id, + ordinal, + member.repositoryKey, + member.repositoryId, + member.snapshotId, + member.projectionDigest, + ], + ); + yield* sql.unsafe('DELETE FROM routing_projection_retirements WHERE projection_digest = ?', [ + member.projectionDigest, + ]); + } }), ); - for (let ordinal = 0; ordinal < identity.members.length; ordinal += 1) { - const member = identity.members[ordinal]!; - yield* sql.withTransaction( - sql.unsafe( - `INSERT INTO workset_generation_members ( - generation_id, ordinal, repository_key, repository_id, snapshot_id, projection_digest - ) VALUES (?, ?, ?, ?, ?, ?)`, - [ - identity.id, - ordinal, - member.repositoryKey, - member.repositoryId, - member.snapshotId, - member.projectionDigest, - ], - ), - ); - } - return { + const receipt = { digest: identity.digest, id: identity.id, manifestDigest: input.manifestDigest, @@ -445,10 +650,50 @@ export const stageCodeGraphWorksetCatalogGenerationFromReceipts = Effect.fn( state: 'staging' as const, worksetName: input.worksetName, }; + return receipt; }), ); }); +/** @internal Retire one serialized prepare attempt without cascading heavy payload. */ +export const retireCodeGraphWorksetCatalogPreparation = Effect.fn('codeGraphWorksetCatalog.retirePreparation')( + function* ( + threadnoteHome: string, + input: {readonly generationId?: string; readonly projectionDigests: readonly string[]}, + ) { + yield* validateInput(() => { + if (input.generationId !== undefined && !GENERATION_ID.test(input.generationId)) { + throw invalid('Workset catalog generation identity is invalid.'); + } + if (input.projectionDigests.length > CODE_GRAPH_WORKSET_CATALOG_LIMITS.membersPerGeneration) { + throw invalid('Workset catalog projection retirement exceeds the supported bound.'); + } + const unique = new Set(input.projectionDigests); + if ( + unique.size !== input.projectionDigests.length || + input.projectionDigests.some(digest => !SHA256_HEX.test(digest)) + ) { + throw invalid('Workset catalog projection retirement identity is invalid.'); + } + }); + const now = yield* currentIsoInstant; + yield* withCatalogWriter(threadnoteHome, sql => + sql.withTransaction( + Effect.gen(function* () { + if (input.generationId !== undefined) { + yield* discardStagingGenerationWithSql(sql, input.generationId); + } + for (const projectionDigest of input.projectionDigests) { + yield* queueProjectionRetirement(sql, projectionDigest, now); + } + yield* markQueuedOrphanProjectionsRetiring(sql, input.projectionDigests.length); + yield* dropQueuedReferencedProjections(sql, input.projectionDigests.length); + }), + ), + ); + }, +); + /** Atomically replace one workset's published pointer after validating every staged projection receipt. */ export const publishCodeGraphWorksetCatalogGeneration = Effect.fn('codeGraphWorksetCatalog.publishGeneration')( function* ( @@ -558,6 +803,65 @@ export const publishCodeGraphWorksetCatalogGeneration = Effect.fn('codeGraphWork }, ); +/** + * Remove one manifest-deleted/renamed workset publication without touching any + * repository snapshot. Heavy derived payload is reclaimed in bounded pages. + */ +export const retireCodeGraphWorksetPublication = Effect.fn('codeGraphWorksetCatalog.retirePublication')(function* ( + threadnoteHome: string, + input: {readonly generationId: string; readonly worksetName: string}, +) { + yield* validateInput(() => { + assertInputText(input.worksetName, 'workset name', 256); + if (!GENERATION_ID.test(input.generationId)) throw invalid('Workset catalog generation identity is invalid.'); + }); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); + return yield* withExclusiveFileLock( + fs, + layout.prepareLockPath, + CATALOG_LOCK_OPTIONS, + Effect.gen(function* () { + const retired = yield* withCatalogWriter(threadnoteHome, sql => + sql.withTransaction( + Effect.gen(function* () { + const pointers = yield* sql.unsafe<{readonly generation_id: unknown}>( + `SELECT generation_id FROM published_worksets + WHERE workset_name = ? AND generation_id = ? LIMIT 1`, + [input.worksetName, input.generationId], + ); + if (pointers.length === 0) return false; + const generationId = requiredText(pointers[0]!.generation_id, 'published generation identity'); + yield* sql.unsafe('DELETE FROM published_worksets WHERE workset_name = ? AND generation_id = ?', [ + input.worksetName, + generationId, + ]); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Published workset pointer changed during retirement.')); + } + yield* sql.unsafe( + `UPDATE workset_generations SET state = 'retired' + WHERE id = ? AND workset_name = ? AND state = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM published_worksets AS p WHERE p.generation_id = workset_generations.id + )`, + [generationId, input.worksetName], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Published workset generation changed during retirement.')); + } + return true; + }), + ), + ); + if (!retired) return {cleanupPending: false, retired}; + const maintenance = yield* maintainCodeGraphWorksetCatalogPreparationPage(threadnoteHome); + return {cleanupPending: maintenance.pendingCleanup, retired}; + }), + ).pipe(mapCatalogError('retire workset catalog publication')); +}); + export const readPublishedCodeGraphWorksetCatalogGeneration = Effect.fn( 'codeGraphWorksetCatalog.readPublishedGeneration', )(function* (threadnoteHome: string, worksetName: string) { @@ -1223,6 +1527,11 @@ export const maintainCodeGraphWorksetResultSets = Effect.fn('codeGraphWorksetCat capacityResultSetsDeleted += yield* changes(sql); capacity = yield* resultSetCapacity(sql); } + yield* maintainCatalogRowsWithSql(sql, { + generationLimit: PRODUCTION_MAINTENANCE_LIMIT, + now, + projectionLimit: 0, + }); return { capacityResultSetsDeleted, expiredResultSetsDeleted, @@ -1250,67 +1559,388 @@ export const maintainCodeGraphWorksetCatalog = Effect.fn('codeGraphWorksetCatalo const generationLimit = yield* validateInput(() => retirementLimit(options.generationLimit, 32)); const projectionLimit = yield* validateInput(() => retirementLimit(options.projectionLimit, 32)); const stagingBefore = yield* validateInput(() => optionalIsoInstant(options.stagingBefore)); + const now = yield* currentIsoInstant; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); + return yield* withExclusiveFileLock( + fs, + layout.prepareLockPath, + CATALOG_LOCK_OPTIONS, + withCatalogWriter(threadnoteHome, sql => + sql.withTransaction( + Effect.gen(function* () { + yield* deleteExpiredResultSets(sql, now, CODE_GRAPH_WORKSET_CATALOG_LIMITS.resultSetsMaximum); + const result = yield* maintainCatalogRowsWithSql(sql, { + generationLimit, + now, + projectionLimit, + stagingBefore, + }); + return { + projectionsDeleted: result.projectionsDeleted, + retiredGenerationsDeleted: result.retiredGenerationsDeleted, + stagingGenerationsRetired: result.stagingGenerationsRetired, + }; + }), + ), + ), + ).pipe(mapCatalogError('maintain workset catalog')); +}); + +/** @internal One physically bounded cleanup page for the home-global prepare lock holder. */ +export const maintainCodeGraphWorksetCatalogPreparationPage = Effect.fn( + 'codeGraphWorksetCatalog.maintainPreparationPage', +)(function* (threadnoteHome: string) { + const now = yield* currentIsoInstant; return yield* withCatalogWriter(threadnoteHome, sql => sql.withTransaction( Effect.gen(function* () { - let stagingGenerationsRetired = 0; - if (stagingBefore !== undefined && generationLimit > 0) { - yield* sql.unsafe( - `UPDATE workset_generations - SET state = 'retired' - WHERE id IN ( - SELECT id FROM workset_generations - WHERE state = 'staging' AND created_at < ? - ORDER BY created_at, id - LIMIT ? - )`, - [stagingBefore, generationLimit], + yield* deleteExpiredResultSets(sql, now, CODE_GRAPH_WORKSET_CATALOG_LIMITS.resultSetsMaximum); + return yield* maintainCatalogRowsWithSql(sql, { + generationLimit: PRODUCTION_MAINTENANCE_LIMIT, + now, + projectionLimit: CODE_GRAPH_WORKSET_CATALOG_LIMITS.membersPerGeneration, + stagingBefore: stagingGenerationCutoff(now), + }); + }), + ), + ); +}); + +function drainCodeGraphWorksetCatalogPreparationPages(threadnoteHome: string) { + return Effect.gen(function* () { + let pendingCleanup = true; + while (pendingCleanup) { + const page = yield* maintainCodeGraphWorksetCatalogPreparationPage(threadnoteHome); + pendingCleanup = page.pendingCleanup; + if (pendingCleanup) yield* Effect.yieldNow; + } + }); +} + +function maintainCatalogRowsWithSql( + sql: SqlClient.SqlClient, + input: { + readonly generationLimit: number; + readonly now: string; + readonly projectionLimit: number; + readonly stagingBefore?: string; + }, +) { + return Effect.gen(function* () { + let stagingGenerationsRetired = 0; + if (input.stagingBefore !== undefined && input.generationLimit > 0) { + yield* sql.unsafe( + `UPDATE workset_generations + SET state = 'retired' + WHERE id = ( + SELECT g.id FROM workset_generations AS g + WHERE g.state = 'staging' AND g.created_at < ? + AND NOT EXISTS ( + SELECT 1 FROM published_worksets AS p WHERE p.generation_id = g.id + ) + ORDER BY g.created_at, g.id + LIMIT 1 + )`, + [input.stagingBefore], + ); + stagingGenerationsRetired = yield* changes(sql); + } + let retiredGenerationsDeleted = 0; + let generationWorkPerformed = stagingGenerationsRetired > 0; + if (input.generationLimit > 0 && !generationWorkPerformed) { + const candidates = yield* sql.unsafe<{readonly id: unknown}>( + `SELECT g.id FROM workset_generations AS g + WHERE g.state = 'retired' + AND NOT EXISTS ( + SELECT 1 FROM published_worksets AS p WHERE p.generation_id = g.id + ) + AND ( + g.member_count > 0 + OR EXISTS ( + SELECT 1 FROM workset_generation_members AS m WHERE m.generation_id = g.id + ) + OR EXISTS ( + SELECT 1 FROM cross_repository_bridge_sets AS b WHERE b.generation_id = g.id + ) + OR NOT EXISTS ( + SELECT 1 FROM result_sets AS r WHERE r.generation_id = g.id + ) + ) + ORDER BY g.created_at, g.id + LIMIT 1`, + ); + const candidate = candidates[0]; + if (candidate !== undefined) { + const generationId = requiredText(candidate.id, 'retired generation identity'); + yield* sql.unsafe( + `DELETE FROM cross_repository_bridges + WHERE generation_id = ? AND ordinal IN ( + SELECT ordinal FROM cross_repository_bridges + WHERE generation_id = ? + ORDER BY ordinal + LIMIT ? + )`, + [generationId, generationId, CATALOG_RECLAIM_ROW_BUDGET], + ); + generationWorkPerformed = (yield* changes(sql)) > 0; + if (!generationWorkPerformed) { + const bridgeSets = yield* sql.unsafe<{readonly bridge_bytes: unknown}>( + 'SELECT bridge_bytes FROM cross_repository_bridge_sets WHERE generation_id = ? LIMIT 1', + [generationId], + ); + if (bridgeSets[0] !== undefined) { + const bridgeBytes = requiredInteger(bridgeSets[0].bridge_bytes, 'retired bridge logical bytes'); + yield* sql.unsafe( + `UPDATE catalog_capacity + SET bridge_logical_bytes = bridge_logical_bytes - ? + WHERE singleton = 1 AND bridge_logical_bytes >= ?`, + [bridgeBytes, bridgeBytes], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Bridge capacity receipt is inconsistent during reclamation.')); + } + yield* sql.unsafe('DELETE FROM cross_repository_bridge_sets WHERE generation_id = ?', [generationId]); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Retired bridge set changed during reclamation.')); + } + generationWorkPerformed = true; + } + } + if (!generationWorkPerformed) { + const members = yield* sql.unsafe<{readonly projection_digest: unknown}>( + `SELECT projection_digest FROM workset_generation_members + WHERE generation_id = ? + ORDER BY ordinal + LIMIT ?`, + [generationId, CATALOG_RECLAIM_ROW_BUDGET], ); - stagingGenerationsRetired = yield* changes(sql); + if (members.length > 0) { + for (const member of members) { + yield* queueProjectionRetirement( + sql, + requiredText(member.projection_digest, 'projection digest'), + input.now, + ); + } + yield* sql.unsafe( + `DELETE FROM workset_generation_members + WHERE generation_id = ? AND ordinal IN ( + SELECT ordinal FROM workset_generation_members + WHERE generation_id = ? + ORDER BY ordinal + LIMIT ? + )`, + [generationId, generationId, CATALOG_RECLAIM_ROW_BUDGET], + ); + const membersDeleted = yield* changes(sql); + if (membersDeleted !== members.length) { + return yield* Effect.fail(corrupt('Retired generation members changed during reclamation.')); + } + yield* markQueuedOrphanProjectionsRetiring(sql, members.length); + yield* dropQueuedReferencedProjections(sql, members.length); + generationWorkPerformed = true; + } } - let retiredGenerationsDeleted = 0; - if (generationLimit > 0) { + if (!generationWorkPerformed) { + yield* sql.unsafe('UPDATE workset_generations SET member_count = 0 WHERE id = ? AND state = ?', [ + generationId, + 'retired', + ]); yield* sql.unsafe( `DELETE FROM workset_generations - WHERE id IN ( - SELECT g.id FROM workset_generations AS g - WHERE g.state = 'retired' - AND NOT EXISTS ( - SELECT 1 FROM published_worksets AS p WHERE p.generation_id = g.id - ) - AND NOT EXISTS ( - SELECT 1 FROM result_sets AS r WHERE r.generation_id = g.id - ) - ORDER BY g.created_at, g.id - LIMIT ? - )`, - [generationLimit], + WHERE id = ? AND state = 'retired' + AND NOT EXISTS ( + SELECT 1 FROM result_sets AS r WHERE r.generation_id = workset_generations.id + )`, + [generationId], ); retiredGenerationsDeleted = yield* changes(sql); + generationWorkPerformed = true; } - let projectionsDeleted = 0; - if (projectionLimit > 0) { - yield* sql.unsafe( - `DELETE FROM repository_snapshots - WHERE projection_digest IN ( - SELECT p.projection_digest FROM repository_snapshots AS p - WHERE NOT EXISTS ( - SELECT 1 FROM workset_generation_members AS m - WHERE m.projection_digest = p.projection_digest - ) - ORDER BY p.created_at, p.projection_digest - LIMIT ? - )`, - [projectionLimit], - ); - projectionsDeleted = yield* changes(sql); - } - return {projectionsDeleted, retiredGenerationsDeleted, stagingGenerationsRetired}; - }), - ), + } + } + let projectionsDeleted = 0; + if (input.projectionLimit > 0 && !generationWorkPerformed) { + yield* markQueuedOrphanProjectionsRetiring(sql, 1); + const projectionsMarked = yield* changes(sql); + if (projectionsMarked === 0) { + projectionsDeleted = yield* reclaimOneProjectionPage(sql); + } + } + const pendingCleanup = yield* catalogCleanupPending(sql, input.projectionLimit > 0); + return {pendingCleanup, projectionsDeleted, retiredGenerationsDeleted, stagingGenerationsRetired}; + }); +} + +function reclaimOneProjectionPage(sql: SqlClient.SqlClient) { + return Effect.gen(function* () { + const queued = yield* sql.unsafe<{readonly projection_digest: unknown}>( + `SELECT q.projection_digest FROM routing_projection_retirements AS q + JOIN repository_snapshots AS p ON p.projection_digest = q.projection_digest + WHERE p.state = 'reclaiming' + AND NOT EXISTS ( + SELECT 1 FROM workset_generation_members AS m + WHERE m.projection_digest = q.projection_digest + ) + ORDER BY q.requested_at, q.projection_digest + LIMIT 1`, + ); + if (queued[0] === undefined) return 0; + const projectionDigest = requiredText(queued[0].projection_digest, 'retired projection digest'); + for (const child of [ + ['routing_exact_keys', '(projection_digest, node_id, key_kind, exact_key)', 'node_id, key_kind, exact_key'], + ['routing_lookup_keys', '(projection_digest, node_id, lookup_key)', 'node_id, lookup_key'], + ['routing_terms', '(projection_digest, node_id, term)', 'node_id, term'], + ] as const) { + if ((yield* deleteRoutingChildPage(sql, child[0], child[1], child[2], projectionDigest)) > 0) return 0; + } + yield* sql.unsafe( + `DELETE FROM routing_symbols + WHERE projection_digest = ? AND node_id IN ( + SELECT node_id FROM routing_symbols + WHERE projection_digest = ? + ORDER BY node_id + LIMIT ? + )`, + [projectionDigest, projectionDigest, CATALOG_RECLAIM_ROW_BUDGET], + ); + if ((yield* changes(sql)) > 0) return 0; + const storage = yield* sql.unsafe<{readonly reserved_bytes: unknown}>( + 'SELECT reserved_bytes FROM routing_projection_storage WHERE projection_digest = ? LIMIT 1', + [projectionDigest], + ); + if (storage.length !== 1) { + return yield* Effect.fail(corrupt('Routing projection storage receipt is missing during reclamation.')); + } + const logicalBytes = requiredInteger(storage[0]!.reserved_bytes, 'routing projection reserved bytes'); + yield* sql.unsafe( + `UPDATE catalog_capacity + SET projection_logical_bytes = projection_logical_bytes - ? + WHERE singleton = 1 AND projection_logical_bytes >= ?`, + [logicalBytes, logicalBytes], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail(corrupt('Routing projection capacity receipt is inconsistent.')); + } + yield* sql.unsafe( + `DELETE FROM repository_snapshots + WHERE projection_digest = ? AND state = 'reclaiming' + AND NOT EXISTS ( + SELECT 1 FROM workset_generation_members AS m + WHERE m.projection_digest = repository_snapshots.projection_digest + )`, + [projectionDigest], + ); + const deleted = yield* changes(sql); + if (deleted !== 1) return yield* Effect.fail(corrupt('Retired routing projection changed during reclamation.')); + return deleted; + }); +} + +function discardStagingGenerationWithSql(sql: SqlClient.SqlClient, generationId: string) { + return sql.unsafe( + `UPDATE workset_generations SET state = 'retired' + WHERE id = ? AND state = 'staging' + AND NOT EXISTS ( + SELECT 1 FROM published_worksets AS p WHERE p.generation_id = workset_generations.id + )`, + [generationId], ); -}); +} + +function queueProjectionRetirement(sql: SqlClient.SqlClient, projectionDigest: string, requestedAt: string) { + return sql.unsafe( + `INSERT OR IGNORE INTO routing_projection_retirements (projection_digest, requested_at) + SELECT projection_digest, ? FROM repository_snapshots WHERE projection_digest = ?`, + [requestedAt, projectionDigest], + ); +} + +function markQueuedOrphanProjectionsRetiring(sql: SqlClient.SqlClient, limit: number) { + if (limit === 0) return Effect.void; + return sql.unsafe( + `UPDATE repository_snapshots SET state = 'reclaiming' + WHERE projection_digest IN ( + SELECT q.projection_digest FROM routing_projection_retirements AS q + WHERE NOT EXISTS ( + SELECT 1 FROM workset_generation_members AS m + WHERE m.projection_digest = q.projection_digest + ) + ORDER BY q.requested_at, q.projection_digest + LIMIT ? + ) AND state IN ('ready', 'staging')`, + [limit], + ); +} + +function deleteRoutingChildPage( + sql: SqlClient.SqlClient, + table: 'routing_exact_keys' | 'routing_lookup_keys' | 'routing_terms', + tuple: string, + orderBy: string, + projectionDigest: string, +) { + return Effect.gen(function* () { + yield* sql.unsafe( + `DELETE FROM ${table} + WHERE ${tuple} IN ( + SELECT ${tuple.slice(1, -1)} FROM ${table} + WHERE projection_digest = ? + ORDER BY ${orderBy} + LIMIT ? + )`, + [projectionDigest, CATALOG_RECLAIM_ROW_BUDGET], + ); + return yield* changes(sql); + }); +} + +function catalogCleanupPending(sql: SqlClient.SqlClient, includeProjections: boolean) { + return sql + .unsafe<{readonly count: unknown}>( + `SELECT ( + EXISTS ( + SELECT 1 FROM workset_generations AS g + WHERE g.state = 'retired' + AND NOT EXISTS ( + SELECT 1 FROM published_worksets AS p WHERE p.generation_id = g.id + ) + AND ( + g.member_count > 0 + OR EXISTS ( + SELECT 1 FROM workset_generation_members AS m WHERE m.generation_id = g.id + ) + OR EXISTS ( + SELECT 1 FROM cross_repository_bridge_sets AS b WHERE b.generation_id = g.id + ) + OR NOT EXISTS ( + SELECT 1 FROM result_sets AS r WHERE r.generation_id = g.id + ) + ) + LIMIT 1 + ) + ${ + includeProjections + ? `OR EXISTS ( + SELECT 1 FROM routing_projection_retirements AS q + WHERE NOT EXISTS ( + SELECT 1 FROM workset_generation_members AS m + WHERE m.projection_digest = q.projection_digest + ) + LIMIT 1 + )` + : '' + } + ) AS count`, + ) + .pipe(Effect.map(rows => requiredInteger(rows[0]?.count, 'cleanup pending count') > 0)); +} +function stagingGenerationCutoff(now: string): string { + return new Date(new Date(now).getTime() - STAGING_GENERATION_RETENTION_MILLISECONDS).toISOString(); +} /** Inspect and reconstruct only the disposable catalog when corruption or schema drift is proven. */ export const recoverCodeGraphWorksetCatalog = Effect.fn('codeGraphWorksetCatalog.recover')(function* ( threadnoteHome: string, @@ -1320,20 +1950,25 @@ export const recoverCodeGraphWorksetCatalog = Effect.fn('codeGraphWorksetCatalog const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); return yield* withExclusiveFileLock( fs, - layout.lockPath, + layout.prepareLockPath, CATALOG_LOCK_OPTIONS, - Effect.gen(function* () { - const health = yield* inspectCatalogLayout(fs, layout); - if (health.state === 'ok') return {previousState: health.state, rebuilt: false}; - if (health.state === 'unavailable') { - return yield* Effect.fail( - new CodeGraphWorksetCatalogError('storage', 'The workset catalog is unavailable and was not rebuilt.'), - ); - } - if (health.state !== 'missing') yield* removeCatalogFiles(fs, layout); - yield* initializeCatalogLayout(fs, layout); - return {previousState: health.state, rebuilt: health.state !== 'missing'}; - }), + withExclusiveFileLock( + fs, + layout.lockPath, + CATALOG_LOCK_OPTIONS, + Effect.gen(function* () { + const health = yield* inspectCatalogLayout(fs, layout); + if (health.state === 'ok') return {previousState: health.state, rebuilt: false}; + if (health.state === 'unavailable') { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('storage', 'The workset catalog is unavailable and was not rebuilt.'), + ); + } + if (health.state !== 'missing') yield* removeCatalogFiles(fs, layout); + yield* initializeCatalogLayout(fs, layout); + return {previousState: health.state, rebuilt: health.state !== 'missing'}; + }), + ), ).pipe(mapCatalogError('recover workset catalog')); }); @@ -1346,11 +1981,16 @@ export const rebuildCodeGraphWorksetCatalog = Effect.fn('codeGraphWorksetCatalog const layout = codeGraphWorksetCatalogLayout(path, threadnoteHome); return yield* withExclusiveFileLock( fs, - layout.lockPath, + layout.prepareLockPath, CATALOG_LOCK_OPTIONS, - Effect.gen(function* () { - yield* removeCatalogFiles(fs, layout); - yield* initializeCatalogLayout(fs, layout); - }), + withExclusiveFileLock( + fs, + layout.lockPath, + CATALOG_LOCK_OPTIONS, + Effect.gen(function* () { + yield* removeCatalogFiles(fs, layout); + yield* initializeCatalogLayout(fs, layout); + }), + ), ).pipe(mapCatalogError('rebuild workset catalog')); }); diff --git a/src/code_graph/workset_catalog/store_support.ts b/src/code_graph/workset_catalog/store_support.ts index 8f50fbba..b9eebcc4 100644 --- a/src/code_graph/workset_catalog/store_support.ts +++ b/src/code_graph/workset_catalog/store_support.ts @@ -22,8 +22,10 @@ import { import {codeGraphWorksetResultSequenceDigest, type PreparedCodeGraphWorksetResultSequenceV1} from './result_set.js'; import {codeGraphWorksetRoutingExactKeys} from './routing_normalization.js'; import { + CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES, configureCodeGraphWorksetCatalogReadConnection, initializeCodeGraphWorksetCatalogSchema, + inspectCodeGraphWorksetCatalogPageSize, inspectCodeGraphWorksetCatalogSchemaVersion, } from './schema.js'; import { @@ -35,7 +37,6 @@ import { type CodeGraphWorksetCatalogHealthV1, type CodeGraphWorksetRoutingProjectionReceiptV1, type CodeGraphWorksetRoutingProjectionDigestStateV1, - type CodeGraphWorksetRoutingProjectionV1, type CodeGraphWorksetRoutingSymbolV1, type CodeGraphWorksetRoutingTermV1, } from './types.js'; @@ -170,12 +171,22 @@ export function withCatalogWriter( }), ); yield* fs.chmod(layout.databasePath, 0o600); + yield* removeObsoleteCatalogV2Files(fs, path, threadnoteHome); return result; }), ); }).pipe(mapCatalogError('write workset catalog')); } +function removeObsoleteCatalogV2Files(fs: FileSystem.FileSystem, path: Path.Path, threadnoteHome: string) { + const legacyDatabase = path.join(threadnoteHome, 'indexes', 'code-graph', 'worksets', 'catalog-v2.sqlite'); + return Effect.forEach( + [legacyDatabase, `${legacyDatabase}-journal`, `${legacyDatabase}-shm`, `${legacyDatabase}-wal`], + candidate => fs.remove(candidate, {force: true}), + {concurrency: 1, discard: true}, + ); +} + export function withCatalogReader( threadnoteHome: string, use: (sql: SqlClient.SqlClient) => Effect.Effect, @@ -235,33 +246,76 @@ export function projectionState(sql: SqlClient.SqlClient, projectionDigest: stri return selectProjectionByDigest(sql, projectionDigest).pipe(Effect.map(row => row?.state)); } +export function dropQueuedReferencedProjections(sql: SqlClient.SqlClient, limit: number) { + if (limit === 0) return Effect.void; + return sql.unsafe( + `DELETE FROM routing_projection_retirements + WHERE projection_digest IN ( + SELECT q.projection_digest FROM routing_projection_retirements AS q + WHERE EXISTS ( + SELECT 1 FROM workset_generation_members AS m + WHERE m.projection_digest = q.projection_digest + ) + ORDER BY q.requested_at, q.projection_digest + LIMIT ? + )`, + [limit], + ); +} + export function insertProjectionHeader( sql: SqlClient.SqlClient, receipt: CodeGraphWorksetRoutingProjectionReceiptV1, now: string, + reservedLogicalBytes: number, + stagingToken: string, ) { return sql.withTransaction( - sql.unsafe( - `INSERT INTO repository_snapshots ( - projection_digest, repository_id, checkout_id, worktree_id, snapshot_id, - snapshot_digest, commit_id, extractor_generation, projector_version, - component_count, symbol_count, state, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staging', ?)`, - [ - receipt.projectionDigest, - receipt.repositoryId, - receipt.checkoutId, - receipt.worktreeId, - receipt.snapshotId, - receipt.snapshotDigest, - receipt.commitId, - receipt.extractorGeneration, - receipt.projectorVersion, - receipt.componentCount, - receipt.symbolCount, - now, - ], - ), + Effect.gen(function* () { + yield* sql.unsafe( + `INSERT INTO repository_snapshots ( + projection_digest, repository_id, checkout_id, worktree_id, snapshot_id, + snapshot_digest, commit_id, extractor_generation, projector_version, + component_count, symbol_count, state, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staging', ?)`, + [ + receipt.projectionDigest, + receipt.repositoryId, + receipt.checkoutId, + receipt.worktreeId, + receipt.snapshotId, + receipt.snapshotDigest, + receipt.commitId, + receipt.extractorGeneration, + receipt.projectorVersion, + receipt.componentCount, + receipt.symbolCount, + now, + ], + ); + yield* sql.unsafe( + `UPDATE catalog_capacity + SET projection_logical_bytes = projection_logical_bytes + ? + WHERE singleton = 1 AND projection_logical_bytes <= ? - bridge_logical_bytes`, + [reservedLogicalBytes, CODE_GRAPH_WORKSET_CATALOG_LIMITS.catalogPhysicalBytesMaximum - reservedLogicalBytes], + ); + if ((yield* changes(sql)) !== 1) { + return yield* Effect.fail( + new CodeGraphWorksetCatalogError('capacity', 'The home-global routing projection catalog is full.'), + ); + } + yield* sql.unsafe( + `INSERT INTO routing_projection_storage ( + projection_digest, logical_bytes, reserved_bytes, staging_token + ) VALUES (?, 0, ?, ?)`, + [receipt.projectionDigest, reservedLogicalBytes, stagingToken], + ); + yield* sql.unsafe( + `INSERT INTO routing_projection_retirements (projection_digest, requested_at) + VALUES (?, ?)`, + [receipt.projectionDigest, now], + ); + }), ); } @@ -283,91 +337,6 @@ export function projectionReceipt( }; } -export function stageProjection( - sql: SqlClient.SqlClient, - projection: CodeGraphWorksetRoutingProjectionV1, - now: string, -) { - return Effect.gen(function* () { - const existing = yield* sql.unsafe( - `SELECT projection_digest, repository_id, checkout_id, worktree_id, snapshot_id, - snapshot_digest, commit_id, extractor_generation, projector_version, - component_count, symbol_count, state - FROM repository_snapshots - WHERE checkout_id = ? AND worktree_id = ? AND snapshot_id = ? AND projector_version = ? - LIMIT 1`, - [projection.checkoutId, projection.worktreeId, projection.snapshotId, projection.projectorVersion], - ); - if (existing.length === 1) { - const metadata = yield* decodeProjectionMetadata(existing[0]!); - if (metadata.projection_digest !== projection.projectionDigest) { - return yield* Effect.fail( - new CodeGraphWorksetCatalogError( - 'invalid-input', - 'A ready snapshot produced different records for the same projector version.', - ), - ); - } - if (metadata.state === 'ready') { - yield* loadAndValidateProjection(sql, projection.projectionDigest, true); - return; - } - yield* sql.withTransaction( - sql.unsafe('DELETE FROM repository_snapshots WHERE projection_digest = ? AND state = ?', [ - projection.projectionDigest, - 'staging', - ]), - ); - } - yield* sql.withTransaction( - sql.unsafe( - `INSERT INTO repository_snapshots ( - projection_digest, repository_id, checkout_id, worktree_id, snapshot_id, - snapshot_digest, commit_id, extractor_generation, projector_version, - component_count, symbol_count, state, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staging', ?)`, - [ - projection.projectionDigest, - projection.repositoryId, - projection.checkoutId, - projection.worktreeId, - projection.snapshotId, - projection.snapshotDigest, - projection.commitId, - projection.extractorGeneration, - projection.projectorVersion, - projection.componentCount, - projection.symbols.length, - now, - ], - ), - ); - for (let offset = 0; offset < projection.symbols.length; offset += PROJECTION_INSERT_BATCH_SIZE) { - const page = projection.symbols.slice(offset, offset + PROJECTION_INSERT_BATCH_SIZE); - yield* sql.withTransaction( - Effect.forEach(page, symbol => insertRoutingSymbol(sql, projection.projectionDigest, symbol), { - concurrency: 1, - discard: true, - }), - ); - } - const stored = yield* loadAndValidateProjection(sql, projection.projectionDigest); - if (stored.receipt.projectionDigest !== projection.projectionDigest) { - return yield* Effect.fail(corrupt('Staged routing projection changed before publication.')); - } - yield* sql.withTransaction( - sql.unsafe( - `UPDATE repository_snapshots SET state = 'ready' - WHERE projection_digest = ? AND state = 'staging'`, - [projection.projectionDigest], - ), - ); - if ((yield* changes(sql)) !== 1) { - return yield* Effect.fail(corrupt('Routing projection publication lost its staging state.')); - } - }); -} - export function insertRoutingSymbol( sql: SqlClient.SqlClient, projectionDigest: string, @@ -557,7 +526,9 @@ export function loadAndValidateProjection(sql: SqlClient.SqlClient, projectionDi export function decodeProjectionMetadata(row: ProjectionRow) { return validateStored(() => { const state = requiredText(row.state, 'projection state'); - if (state !== 'ready' && state !== 'staging') throw corrupt('Routing projection state is invalid.'); + if (state !== 'ready' && state !== 'reclaiming' && state !== 'staging') { + throw corrupt('Routing projection state is invalid.'); + } const projectionDigest = requiredText(row.projection_digest, 'projection digest'); const repositoryId = requiredText(row.repository_id, 'repository identity'); const checkoutId = requiredText(row.checkout_id, 'checkout identity'); @@ -1041,6 +1012,10 @@ export function inspectCatalogLayout( if (schemaVersion !== CODE_GRAPH_WORKSET_CATALOG_SCHEMA_VERSION) { return {schemaVersion: schemaVersion ?? 0, state: 'incompatible'} as const; } + const pageSize = yield* inspectCodeGraphWorksetCatalogPageSize(sql); + if (pageSize !== CODE_GRAPH_WORKSET_CATALOG_PAGE_SIZE_BYTES) { + return {schemaVersion, state: 'incompatible'} as const; + } const quick = yield* sql.unsafe<{readonly quick_check: unknown}>('PRAGMA quick_check'); if (quick.length !== 1 || quick[0]?.quick_check !== 'ok') { return {detail: 'SQLite integrity validation failed.', state: 'corrupt'} as const; diff --git a/src/code_graph/workset_catalog/types.ts b/src/code_graph/workset_catalog/types.ts index a494eaf3..67c823a9 100644 --- a/src/code_graph/workset_catalog/types.ts +++ b/src/code_graph/workset_catalog/types.ts @@ -4,10 +4,14 @@ export const CODE_GRAPH_WORKSET_CATALOG_PROJECTOR_VERSION = 2 as const; export const CODE_GRAPH_WORKSET_CATALOG_LIMITS = { bridgeRecordBytesMaximum: 64 * 1_024, + bridgeSetBytesMaximum: 64 * 1_024 * 1_024, bridgesPerGeneration: 250_000, + catalogPhysicalBytesMaximum: 4 * 1_024 * 1_024 * 1_024, exactKeysPerSymbol: 256, lookupKeysPerSymbol: 64, membersPerGeneration: 4_096, + projectionBytesMaximum: 128 * 1_024 * 1_024, + projectionPageBytesMaximum: 8 * 1_024 * 1_024, readPageMaximum: 1_000, resultSetBytesMaximum: 2 * 1_024 * 1_024, resultSetCardBytesMaximum: 64 * 1_024, diff --git a/src/code_graph/workset_catalog/workset.ts b/src/code_graph/workset_catalog/workset.ts index b2c06ab9..2e6c4c97 100644 --- a/src/code_graph/workset_catalog/workset.ts +++ b/src/code_graph/workset_catalog/workset.ts @@ -1,5 +1,6 @@ -import {Effect, FileSystem, Result} from 'effect'; +import {Effect, Exit, FileSystem, Path, Result} from 'effect'; import {sha256HexSync} from '../../crypto/sha256.js'; +import {isFileLockTimeout, withExclusiveFileLock} from '../../effect/file_lock.js'; import {requireWorkset} from '../../manifest.js'; import type {ProjectManifest, ResolvedWorkset, RuntimeConfig} from '../../types.js'; import {expandPath} from '../../utils.js'; @@ -23,12 +24,16 @@ import { type RepositoryIdentity, } from '../types.js'; import {stageCodeGraphWorksetRoutingProjectionScoped} from './projection_builder.js'; +import {codeGraphWorksetCatalogLayout} from './layout.js'; import { + maintainCodeGraphWorksetCatalogPreparationPage, publishCodeGraphWorksetCatalogGeneration, readPublishedCodeGraphWorksetCatalogGeneration, registerCodeGraphQualifiedRef, + retireCodeGraphWorksetCatalogPreparation, stageCodeGraphWorksetCatalogGenerationFromReceipts, } from './store.js'; +import {CodeGraphWorksetCatalogError} from './types.js'; import type { CodeGraphWorksetCatalogGenerationDigestMemberV1, CodeGraphWorksetCatalogGenerationReceiptV1, @@ -38,6 +43,12 @@ import type { export const CODE_GRAPH_WORKSET_PREPARE_CONCURRENCY_DEFAULT = 2; export const CODE_GRAPH_WORKSET_PREPARE_CONCURRENCY_MAXIMUM = 8; +const WORKSET_PREPARE_LOCK_OPTIONS = { + heartbeatIntervalMilliseconds: 10_000, + retryIntervalMilliseconds: 25, + staleAfterMilliseconds: 30_000, + waitTimeoutMilliseconds: 30_000, +} as const; export type CodeGraphWorksetPrepareMemberV1 = | { @@ -170,11 +181,15 @@ const prepareCodeGraphWorksetScoped = Effect.fn('codeGraphWorkset.prepareScoped' options: PrepareCodeGraphWorksetOptionsV1 = {}, ) { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const indexer = yield* CodeGraphIndexer; const workset = yield* requireWorkset(config.manifestPath, worksetName); const concurrency = yield* Effect.try({ try: () => prepareConcurrency(options.concurrency), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => + new CodeGraphWorksetCatalogError('invalid-input', cause instanceof Error ? cause.message : String(cause), { + cause, + }), }); const manifestDigest = codeGraphWorksetManifestDigest(workset); const indexed = yield* Effect.forEach( @@ -182,45 +197,111 @@ const prepareCodeGraphWorksetScoped = Effect.fn('codeGraphWorkset.prepareScoped' project => prepareConfiguredSnapshot(config, project, fs, indexer), {concurrency}, ); - // Projection reads and catalog appends are deliberately serial: at most one - // normalized symbol page is live across the complete preparation. - const configured: readonly PreparedMemberWithProjection[] = yield* Effect.forEach( - indexed, - member => stageConfiguredMember(config, member), - {concurrency: 1}, - ); - const unresolved: readonly PreparedMemberWithProjection[] = workset.unresolvedProjects.map( - project => - ({ - project: safeLabel(project), - reason: 'unknown-project', - state: 'excluded', - }) as const satisfies CodeGraphWorksetPrepareMemberV1, + const projectionDigests = new Set(); + let stagedGenerationId: string | undefined; + const critical = Effect.gen(function* () { + yield* assertCurrentWorksetManifest(config, workset.name, manifestDigest); + yield* drainCodeGraphWorksetCatalogCleanup(config.agentContextHome); + // Projection reads and catalog appends are deliberately serial: at most one + // normalized symbol page is live across the complete preparation. + const configured: readonly PreparedMemberWithProjection[] = yield* Effect.forEach( + indexed, + member => + stageConfiguredMember(config, member).pipe( + Effect.tap(prepared => + Effect.sync(() => { + if (prepared.state === 'ready') projectionDigests.add(prepared.projectionDigest); + }), + ), + ), + {concurrency: 1}, + ); + const unresolved: readonly PreparedMemberWithProjection[] = workset.unresolvedProjects.map( + project => + ({ + project: safeLabel(project), + reason: 'unknown-project', + state: 'excluded', + }) as const satisfies CodeGraphWorksetPrepareMemberV1, + ); + const members: readonly PreparedMemberWithProjection[] = [...configured, ...unresolved]; + const generationMembers = members.flatMap(member => + member.state === 'ready' ? [preparedGenerationMember(member)] : [], + ); + if (generationMembers.length === 0) { + return prepareResult(workset.name, manifestDigest, members, undefined); + } + yield* assertPreparedMemberLeases(members); + const staged = yield* stageCodeGraphWorksetCatalogGenerationFromReceipts(config.agentContextHome, { + manifestDigest, + members: generationMembers, + worksetName: workset.name, + }); + stagedGenerationId = staged.state === 'staging' ? staged.id : undefined; + const bridgeMembers = members.filter((member): member is PreparedReadyMember => member.state === 'ready'); + const bridges = yield* prepareCodeGraphWorksetBridgesForGeneration(config, staged.id, bridgeMembers); + yield* assertPreparedMemberLeases(members); + const published = yield* publishCodeGraphWorksetCatalogGeneration(config.agentContextHome, { + beforePointerSwap: () => + assertPreparedMemberLeases(members).pipe( + Effect.andThen(assertCurrentWorksetManifest(config, workset.name, manifestDigest)), + ), + generationId: staged.id, + worksetName: workset.name, + }); + stagedGenerationId = undefined; + return prepareResult(workset.name, manifestDigest, members, published, bridges); + }).pipe( + Effect.onExit(exit => + (Exit.isSuccess(exit) + ? drainCodeGraphWorksetCatalogCleanup(config.agentContextHome) + : retireCodeGraphWorksetCatalogPreparation(config.agentContextHome, { + ...(stagedGenerationId === undefined ? {} : {generationId: stagedGenerationId}), + projectionDigests: [...projectionDigests], + }).pipe(Effect.andThen(drainCodeGraphWorksetCatalogCleanup(config.agentContextHome))) + ).pipe(Effect.catchCause(() => Effect.void)), + ), ); - const members: readonly PreparedMemberWithProjection[] = [...configured, ...unresolved]; - const generationMembers = members.flatMap(member => - member.state === 'ready' ? [preparedGenerationMember(member)] : [], + const layout = codeGraphWorksetCatalogLayout(path, config.agentContextHome); + return yield* withExclusiveFileLock(fs, layout.prepareLockPath, WORKSET_PREPARE_LOCK_OPTIONS, critical).pipe( + Effect.mapError(cause => + cause instanceof CodeGraphWorksetCatalogError + ? cause + : new CodeGraphWorksetCatalogError( + isFileLockTimeout(cause) ? 'busy' : 'storage', + isFileLockTimeout(cause) + ? 'Timed out waiting to prepare the home-global workset catalog.' + : 'Unable to serialize home-global workset preparation.', + {cause}, + ), + ), ); - if (generationMembers.length === 0) { - return prepareResult(workset.name, manifestDigest, members, undefined); - } - yield* assertPreparedMemberLeases(members); - const staged = yield* stageCodeGraphWorksetCatalogGenerationFromReceipts(config.agentContextHome, { - manifestDigest, - members: generationMembers, - worksetName: workset.name, - }); - const bridgeMembers = members.filter((member): member is PreparedReadyMember => member.state === 'ready'); - const bridges = yield* prepareCodeGraphWorksetBridgesForGeneration(config, staged.id, bridgeMembers); - yield* assertPreparedMemberLeases(members); - const published = yield* publishCodeGraphWorksetCatalogGeneration(config.agentContextHome, { - beforePointerSwap: () => assertPreparedMemberLeases(members), - generationId: staged.id, - worksetName: workset.name, - }); - return prepareResult(workset.name, manifestDigest, members, published, bridges); }); +function assertCurrentWorksetManifest(config: RuntimeConfig, worksetName: string, expectedDigest: string) { + return requireWorkset(config.manifestPath, worksetName).pipe( + Effect.flatMap(current => + codeGraphWorksetManifestDigest(current) === expectedDigest + ? Effect.void + : Effect.fail( + new CodeGraphWorksetCatalogError( + 'stale', + 'The workset definition changed while its catalog generation was preparing.', + ), + ), + ), + Effect.mapError(cause => + cause instanceof CodeGraphWorksetCatalogError + ? cause + : new CodeGraphWorksetCatalogError( + 'stale', + 'The workset definition changed while its catalog generation was preparing.', + {cause}, + ), + ), + ); +} + export const prepareCodeGraphWorkset = Effect.fn('codeGraphWorkset.prepare')(function* ( config: RuntimeConfig, worksetName: string, @@ -551,7 +632,10 @@ export const prepareCodeGraphWorksetBridgesForGeneration = Effect.fn('codeGraphW const resolution = yield* Effect.result( Effect.try({ try: () => resolveCodeGraphCrossRepositoryBridges(repositories), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => + new CodeGraphWorksetCatalogError('invalid-input', cause instanceof Error ? cause.message : String(cause), { + cause, + }), }), ); if (Result.isFailure(resolution)) { @@ -646,6 +730,16 @@ function assertBridgeMemberLeases(members: readonly CodeGraphWorksetBridgePrepar }); } +function drainCodeGraphWorksetCatalogCleanup(threadnoteHome: string) { + return Effect.gen(function* () { + for (;;) { + const page = yield* maintainCodeGraphWorksetCatalogPreparationPage(threadnoteHome); + if (!page.pendingCleanup) return; + yield* Effect.yieldNow; + } + }); +} + function statusStateCounts( members: readonly CodeGraphWorksetStatusMemberV1[], ): Record { diff --git a/src/code_graph/workset_query_v2.ts b/src/code_graph/workset_query_v2.ts index 885d890e..06dfd733 100644 --- a/src/code_graph/workset_query_v2.ts +++ b/src/code_graph/workset_query_v2.ts @@ -1,4 +1,3 @@ -// oxlint-disable effecttsgo/lazy-effect -- Injected functions construct fresh clock and routing effects per invocation. import {Clock, Effect, FileSystem} from 'effect'; import {readSeedManifest, requireWorkset} from '../manifest.js'; import type {ProjectManifest, RuntimeConfig} from '../types.js'; @@ -113,7 +112,7 @@ export interface CodeGraphWorksetQueryV2DependenciesV1 { readonly deepQuery: ( repository: CodeGraphWorksetRouterRepositoryCandidateV1, ) => Effect.Effect; - readonly nowMilliseconds: () => Effect.Effect; + readonly nowMilliseconds: Effect.Effect; readonly persist: ( result: CodeGraphWorksetQueryResultV2, qualifiedRefs: readonly QualifiedCodeGraphRefV1[], @@ -121,7 +120,7 @@ export interface CodeGraphWorksetQueryV2DependenciesV1 { readonly readBridgeExpansion: ( router: CodeGraphWorksetRouterResultV1, ) => Effect.Effect; - readonly route: () => Effect.Effect; + readonly route: Effect.Effect; } interface CodeGraphWorksetQueryV2TimingV1 { @@ -166,13 +165,13 @@ export const runCodeGraphWorksetQueryV2Core = Effect.fn('codeGraphWorksetV2.runC timing: CodeGraphWorksetQueryV2TimingV1 = {}, ) { const prepared = validateCoreInput(input); - const observedStarted = yield* dependencies.nowMilliseconds(); + const observedStarted = yield* dependencies.nowMilliseconds; const requestedStarted = timing.startedAtMilliseconds; if (requestedStarted !== undefined && (!Number.isSafeInteger(requestedStarted) || requestedStarted < 0)) { throw new Error('The deadline clock origin is invalid.'); } const started = Math.min(observedStarted, requestedStarted ?? observedStarted); - const catalogRouter = yield* dependencies.route(); + const catalogRouter = yield* dependencies.route; validateRouterReceipt(prepared, catalogRouter); const bridgeExpansion = yield* dependencies.readBridgeExpansion(catalogRouter); const router = expandCodeGraphWorksetRouterWithBridges(catalogRouter, prepared.published, bridgeExpansion); @@ -187,7 +186,7 @@ export const runCodeGraphWorksetQueryV2Core = Effect.fn('codeGraphWorksetV2.runC let stopReason: WorksetCoverageV2['stopReason'] = 'exhaustion'; for (;;) { - const now = yield* dependencies.nowMilliseconds(); + const now = yield* dependencies.nowMilliseconds; const remainingMilliseconds = Math.max(0, prepared.deadlineMilliseconds - Math.max(0, now - started)); const expansion = selectCodeGraphWorksetAdaptiveExpansionBatch({ alreadySelectedRepositoryKeys: selectedRepositoryKeys, @@ -210,7 +209,7 @@ export const runCodeGraphWorksetQueryV2Core = Effect.fn('codeGraphWorksetV2.runC return Effect.succeed({repository, state: 'skipped' as const}); } return Effect.gen(function* () { - const taskStarted = yield* dependencies.nowMilliseconds(); + const taskStarted = yield* dependencies.nowMilliseconds; const taskRemaining = Math.max(0, prepared.deadlineMilliseconds - Math.max(0, taskStarted - started)); if (taskRemaining === 0) return {repository, state: 'timed-out' as const}; attemptedRepositoryKeys.add(repository.repositoryKey); @@ -241,7 +240,7 @@ export const runCodeGraphWorksetQueryV2Core = Effect.fn('codeGraphWorksetV2.runC break; } - const afterBatch = yield* dependencies.nowMilliseconds(); + const afterBatch = yield* dependencies.nowMilliseconds; if (Math.max(0, afterBatch - started) >= prepared.deadlineMilliseconds) { stopReason = 'deadline'; break; @@ -387,7 +386,7 @@ export const executeCodeGraphWorksetV2 = Effect.fn('codeGraphWorksetV2.execute') threadnoteHome: config.agentContextHome, }); }, - nowMilliseconds: () => Clock.currentTimeMillis, + nowMilliseconds: Clock.currentTimeMillis, persist: (result, qualifiedRefs) => Effect.gen(function* () { yield* Effect.forEach(qualifiedRefs, ref => registerCodeGraphQualifiedRef(config.agentContextHome, ref), { @@ -410,12 +409,11 @@ export const executeCodeGraphWorksetV2 = Effect.fn('codeGraphWorksetV2.execute') } satisfies CodeGraphWorksetQueryBridgeExpansionV1), ), ), - route: () => - routeCodeGraphWorksetCatalogCandidates(source, { - limits: {repositoryLimit: Math.min(512, Math.max(64, runtime.input.members.length))}, - query: options.query, - worksetName: runtime.input.worksetName, - }), + route: routeCodeGraphWorksetCatalogCandidates(source, { + limits: {repositoryLimit: Math.min(512, Math.max(64, runtime.input.members.length))}, + query: options.query, + worksetName: runtime.input.worksetName, + }), }, runtime.input, {startedAtMilliseconds: deadlineStartedAtMilliseconds}, diff --git a/src/code_graph/worktree_reconciliation.ts b/src/code_graph/worktree_reconciliation.ts index ad92a4eb..513b128f 100644 --- a/src/code_graph/worktree_reconciliation.ts +++ b/src/code_graph/worktree_reconciliation.ts @@ -1,4 +1,4 @@ -import {Crypto, Effect, FileSystem, Path} from 'effect'; +import {Crypto, Effect, FileSystem, Path, Schema} from 'effect'; import {CommandExecutor} from '../effect/command.js'; import {SystemInfo} from '../effect/system.js'; import { @@ -31,6 +31,11 @@ export {type CodeGraphWorktreeReconciliationCandidate} from './store.js'; export const CODE_GRAPH_WORKTREE_RECONCILIATION_CANDIDATE_LIMIT = 32; +class CodeGraphWorktreeAuthorityChanged extends Schema.TaggedErrorClass()( + 'CodeGraphWorktreeAuthorityChanged', + {message: Schema.String}, +) {} + export interface CodeGraphWorktreeReconciliationAuthorityInput { readonly anchorMatches: boolean; readonly evidenceStable: boolean; @@ -392,7 +397,7 @@ export const makeCodeGraphWorktreeReconciler = Effect.fn('codeGraph.makeWorktree }), ); return locked; - }).pipe(Effect.catch(() => Effect.succeed({reason: 'catalog-unavailable', state: 'deferred'} as const))); + }); return {tick} satisfies CodeGraphWorktreeReconcilerShape; }), @@ -417,7 +422,11 @@ export const makeLiveCodeGraphWorktreeReconciler = Effect.fn('codeGraph.makeLive Effect.gen(function* () { const inspected = yield* provideLive(inspectCodeGraphViewDatabaseTarget(input.threadnoteHome, input.checkoutId)); if (inspected.state !== 'ready' || inspected.databasePath !== input.databasePath) { - return yield* Effect.fail(new Error(`Code graph database target changed before ${operation}.`)); + return yield* Effect.fail( + new CodeGraphWorktreeAuthorityChanged({ + message: `Code graph database target changed before ${operation}.`, + }), + ); } if (yield* provideLive(codeGraphMaintenanceIntentActive(input.threadnoteHome))) { return yield* Effect.fail(new CodeGraphMaintenanceActiveError()); diff --git a/src/command-shim.ts b/src/command-shim.ts index d95b2961..15228166 100644 --- a/src/command-shim.ts +++ b/src/command-shim.ts @@ -201,6 +201,5 @@ function cmdQuote(value: string): string { function pathEntryExists(fs: FileSystem.FileSystem, target: string): Effect.Effect { return Effect.all([fs.stat(target).pipe(Effect.option), fs.readLink(target).pipe(Effect.option)]).pipe( Effect.map(([info, link]) => Option.isSome(info) || Option.isSome(link)), - Effect.catch(() => Effect.succeed(false)), ); } diff --git a/src/cursor-plugin.ts b/src/cursor-plugin.ts index edb9cb8c..c5cb5faa 100644 --- a/src/cursor-plugin.ts +++ b/src/cursor-plugin.ts @@ -4,6 +4,10 @@ import {SystemInfo} from './effect/system.js'; import type {DoctorCheck} from './types.js'; import {errorMessage, expandPath, findExecutable, readFileIfExists, toolRoot} from './utils.js'; +class CursorPluginError extends Error { + readonly _tag = 'CursorPluginError' as const; +} + const CURSOR_PLUGIN_NAME = 'threadnote'; const CURSOR_PLUGIN_MANIFEST = '.cursor-plugin/plugin.json'; const CURSOR_PLUGIN_RULE = 'rules/threadnote.mdc'; @@ -119,7 +123,7 @@ const inspectCursorPluginRoot = Effect.fn('cursorPlugin.inspectRoot')(function* const path = yield* Path.Path; const manifestPath = path.join(pluginRoot, CURSOR_PLUGIN_MANIFEST); const manifest = yield* readCursorPluginManifest(manifestPath).pipe( - Effect.mapError(cause => new Error(`${manifestPath}: ${errorMessage(cause)}`)), + Effect.mapError(cause => new CursorPluginError(`${manifestPath}: ${errorMessage(cause)}`)), ); if (manifest.name !== CURSOR_PLUGIN_NAME) { return { @@ -151,7 +155,9 @@ const inspectCursorPluginRoot = Effect.fn('cursorPlugin.inspectRoot')(function* const bundledManifest = yield* readCursorPluginManifest(path.join(bundledRoot, CURSOR_PLUGIN_MANIFEST)); const bundledRule = yield* readFileIfExists(path.join(bundledRoot, CURSOR_PLUGIN_RULE)); if (bundledRule === undefined) { - return yield* Effect.fail(new Error('The standalone release is missing its bundled Cursor plugin rule.')); + return yield* Effect.fail( + new CursorPluginError('The standalone release is missing its bundled Cursor plugin rule.'), + ); } const comparison = compareSemver(manifest.version, bundledManifest.version); if (comparison < 0) { @@ -177,10 +183,10 @@ const inspectCursorPluginRoot = Effect.fn('cursorPlugin.inspectRoot')(function* const readCursorPluginManifest = Effect.fn('cursorPlugin.readManifest')(function* (manifestPath: string) { const raw = yield* readFileIfExists(manifestPath); - if (raw === undefined) return yield* Effect.fail(new Error('manifest is missing')); + if (raw === undefined) return yield* Effect.fail(new CursorPluginError('manifest is missing')); const parsed = yield* Effect.try({ try: () => JSON.parse(raw) as unknown, - catch: cause => new Error('manifest is not valid JSON', {cause}), + catch: cause => new CursorPluginError('manifest is not valid JSON', {cause}), }); if ( typeof parsed !== 'object' || @@ -190,7 +196,7 @@ const readCursorPluginManifest = Effect.fn('cursorPlugin.readManifest')(function typeof (parsed as Record).version !== 'string' || !isSemver((parsed as Record).version as string) ) { - return yield* Effect.fail(new Error('manifest must declare string name and semantic version fields')); + return yield* Effect.fail(new CursorPluginError('manifest must declare string name and semantic version fields')); } return parsed as CursorPluginManifest; }); diff --git a/src/effect/ai/consolidator.ts b/src/effect/ai/consolidator.ts index 4063586d..3aa9eb61 100644 --- a/src/effect/ai/consolidator.ts +++ b/src/effect/ai/consolidator.ts @@ -1,6 +1,6 @@ import * as BunHttpClient from '@effect/platform-bun/BunHttpClient'; import {OpenAiClient, OpenAiLanguageModel} from '@effect/ai-openai-compat'; -import {Context, Effect, Layer, pipe, Redacted, Schema} from 'effect'; +import {Context, Effect, Layer, Redacted, Schema} from 'effect'; import {LanguageModel} from 'effect/unstable/ai'; import type {RuntimeConfig} from '../../types.js'; import {generateWithSelectedLocalModel} from '../../models/inference.js'; @@ -130,7 +130,11 @@ export function effectAiLanguageModelLayer( } export function runEffectAiConsolidation(prompt: string, config: EffectAiConfiguration) { - return pipe(consolidateWithAiEffect(prompt), Effect.provide(aiConsolidatorLayer(config))); + return Effect.scoped( + Layer.build(aiConsolidatorLayer(config)).pipe( + Effect.flatMap(context => consolidateWithAiEffect(prompt).pipe(Effect.provide(context))), + ), + ); } export const runNativeAiConsolidation = Effect.fn('AiConsolidator.consolidateNative')(function* ( diff --git a/src/effect/ai/enrichment.ts b/src/effect/ai/enrichment.ts index f20db7f7..2ae2c2e6 100644 --- a/src/effect/ai/enrichment.ts +++ b/src/effect/ai/enrichment.ts @@ -93,18 +93,24 @@ export const runEffectAiMemoryEnrichment = Effect.fn('MemoryEnricher.run')(funct input: MemoryEnrichmentInput, config: EffectAiConfiguration, ) { - return yield* enrichMemoryEffect(input).pipe( - Effect.provide(memoryEnricherLayer(config)), - Effect.timeoutOrElse({ - duration: MEMORY_ENRICHMENT_TIMEOUT_MILLISECONDS, - orElse: () => - Effect.fail( - new AiMemoryEnrichmentFailed({ - cause: new Error('Memory enrichment timed out.'), - message: 'Effect AI memory enrichment timed out.', + return yield* Effect.scoped( + Layer.build(memoryEnricherLayer(config)).pipe( + Effect.flatMap(context => + enrichMemoryEffect(input).pipe( + Effect.provide(context), + Effect.timeoutOrElse({ + duration: MEMORY_ENRICHMENT_TIMEOUT_MILLISECONDS, + orElse: () => + Effect.fail( + new AiMemoryEnrichmentFailed({ + cause: new Error('Memory enrichment timed out.'), + message: 'Effect AI memory enrichment timed out.', + }), + ), }), ), - }), + ), + ), ); }); diff --git a/src/effect/ai/isolated-local-model-runtime.ts b/src/effect/ai/isolated-local-model-runtime.ts index 32bf7dd7..472f217a 100644 --- a/src/effect/ai/isolated-local-model-runtime.ts +++ b/src/effect/ai/isolated-local-model-runtime.ts @@ -1,4 +1,4 @@ -import {Cause, Effect, Exit, Layer, Option, Schema, Semaphore} from 'effect'; +import {Cause, Effect, Exit, Layer, Option, Schema, Semaphore, Stream} from 'effect'; import type {LocalModelManifest} from '../../models/catalog.js'; import {parseLocalModelManifest} from '../../models/catalog.js'; import { @@ -14,7 +14,6 @@ import { UnsupportedNativeRuntime, } from './errors.js'; import { - localModelRuntimeLayer, LocalModelRuntime, type LocalEmbeddingRequest, type LocalGenerationRequest, @@ -106,7 +105,7 @@ export interface IsolatedLocalModelRuntimeOptions { } interface LocalModelRuntimeWithDiagnostics extends LocalModelRuntimeShape { - readonly diagnostics: () => Effect.Effect; + readonly diagnostics: Effect.Effect; } interface PendingResponse { @@ -157,19 +156,18 @@ export function isolatedLocalModelRuntimeLayer( }; const service: LocalModelRuntimeWithDiagnostics = { - diagnostics: () => - permits.withPermit( - request('diagnostics', {}, decodeDiagnostics).pipe( - Effect.mapError(error => - error instanceof LocalModelWorkerTransportError - ? new NativeRuntimeUnavailable({ - cause: genericWorkerCause(error.reason), - message: `The isolated local AI worker could not report runtime diagnostics: ${error.message}`, - }) - : remoteNativeRuntimeError(error), - ), + diagnostics: permits.withPermit( + request('diagnostics', {}, decodeDiagnostics).pipe( + Effect.mapError(error => + error instanceof LocalModelWorkerTransportError + ? new NativeRuntimeUnavailable({ + cause: genericWorkerCause(error.reason), + message: `The isolated local AI worker could not report runtime diagnostics: ${error.message}`, + }) + : remoteNativeRuntimeError(error), ), ), + ), embedMany: input => permits.withPermit( Effect.gen(function* () { @@ -225,47 +223,71 @@ export function isolatedLocalModelRuntimeLayer( */ export const localModelWorkerServer: Effect.Effect = Effect.gen(function* () { const runtime = yield* LocalModelRuntime; - yield* Effect.callback(resume => { - void serveWorker(runtime, { - input: process.stdin as AsyncIterable, - writeLine: line => - new Promise((resolve, reject) => { - process.stdout.write(`${line}\n`, error => (error ? reject(error) : resolve())); - }), - }).then( - () => resume(Effect.void), - () => resume(Effect.void), - ); - return Effect.sync(() => { - if (!process.stdin.destroyed) process.stdin.pause(); - }); - }); + yield* serveWorker(runtime, { + input: process.stdin as AsyncIterable, + writeLine: line => + new Promise((resolve, reject) => { + process.stdout.write(`${line}\n`, error => (error ? reject(error) : resolve())); + }), + }).pipe( + Effect.catch(() => Effect.void), + Effect.ensuring( + Effect.sync(() => { + if (!process.stdin.destroyed) process.stdin.pause(); + }), + ), + ); }); -export const nativeLocalModelWorkerServer = localModelWorkerServer.pipe(Effect.provide(localModelRuntimeLayer())); - export interface LocalModelWorkerServerIo { readonly input: AsyncIterable; readonly writeLine: (line: string) => Promise; } -export async function serveWorker(runtime: LocalModelRuntimeShape, io: LocalModelWorkerServerIo): Promise { +class LocalModelWorkerServerError extends Error { + readonly _tag = 'LocalModelWorkerServerError' as const; +} + +export function serveWorker( + runtime: LocalModelRuntimeShape, + io: LocalModelWorkerServerIo, +): Effect.Effect { const decoder = new TextDecoder(); let buffered = ''; - for await (const chunk of io.input) { - buffered += typeof chunk === 'string' ? chunk : decoder.decode(chunk, {stream: true}); - for (;;) { - const newline = buffered.indexOf('\n'); - if (newline < 0) break; - const line = buffered.slice(0, newline).replace(/\r$/, ''); - buffered = buffered.slice(newline + 1); - if (!line) continue; - await io.writeLine(JSON.stringify(await handleWorkerLine(runtime, line))); - } - } - buffered += decoder.decode(); - const finalLine = buffered.trim(); - if (finalLine) await io.writeLine(JSON.stringify(await handleWorkerLine(runtime, finalLine))); + const writeResponse = (line: string) => + handleWorkerLine(runtime, line).pipe( + Effect.flatMap(response => + Effect.tryPromise({ + try: () => io.writeLine(JSON.stringify(response)), + catch: cause => new LocalModelWorkerServerError('Could not write local model worker response.', {cause}), + }), + ), + ); + const consumeChunk = (chunk: string | Uint8Array) => + Effect.gen(function* () { + buffered += typeof chunk === 'string' ? chunk : decoder.decode(chunk, {stream: true}); + for (;;) { + const newline = buffered.indexOf('\n'); + if (newline < 0) break; + const line = buffered.slice(0, newline).replace(/\r$/, ''); + buffered = buffered.slice(newline + 1); + if (!line) continue; + yield* writeResponse(line); + } + }); + return Stream.fromAsyncIterable( + io.input, + cause => new LocalModelWorkerServerError('Could not read local model worker input.', {cause}), + ).pipe( + Stream.runForEach(consumeChunk), + Effect.andThen( + Effect.gen(function* () { + buffered += decoder.decode(); + const finalLine = buffered.trim(); + if (finalLine) yield* writeResponse(finalLine); + }), + ), + ); } class LocalModelWorkerPool { @@ -659,33 +681,36 @@ function developmentStandaloneScript(system: SystemInfoShape): Option.Option { +function handleWorkerLine(runtime: LocalModelRuntimeShape, line: string): Effect.Effect { const request = decodeWorkerRequest(line); - if (Option.isNone(request)) return protocolFailure('invalid'); + if (Option.isNone(request)) return Effect.succeed(protocolFailure('invalid')); const effect = withThreadnoteProcessActivity( 'local-model-worker', workerOperationLabel(request.value.operation), dispatchWorkerRequest(runtime, request.value), {idleTransitionDelayMilliseconds: LOCAL_MODEL_PROCESS_ACTIVITY_IDLE_DELAY_MS}, ); - const exit = await Effect.runPromiseExit(effect); - if (Exit.isSuccess(exit)) { - return { - id: request.value.id, - ok: true, - protocol: PROTOCOL_VERSION, - result: exit.value, - }; - } - const failure = Cause.findErrorOption(exit.cause); - return { - error: { - tag: Option.isSome(failure) ? operationErrorTag(failure.value) : 'WorkerOperationFailed', - }, - id: request.value.id, - ok: false, - protocol: PROTOCOL_VERSION, - }; + return Effect.exit(effect).pipe( + Effect.map(exit => { + if (Exit.isSuccess(exit)) { + return { + id: request.value.id, + ok: true, + protocol: PROTOCOL_VERSION, + result: exit.value, + } satisfies WorkerResponse; + } + const failure = Cause.findErrorOption(exit.cause); + return { + error: { + tag: Option.isSome(failure) ? operationErrorTag(failure.value) : 'WorkerOperationFailed', + }, + id: request.value.id, + ok: false, + protocol: PROTOCOL_VERSION, + } satisfies WorkerResponse; + }), + ); } function workerOperationLabel(operation: WorkerOperation): string { @@ -698,7 +723,7 @@ function dispatchWorkerRequest( ): Effect.Effect { if (request.operation === 'diagnostics') { const diagnostics = (runtime as Partial).diagnostics; - return diagnostics ? diagnostics() : Effect.fail({_tag: 'NativeRuntimeUnavailable'}); + return diagnostics ?? Effect.fail({_tag: 'NativeRuntimeUnavailable'}); } if (request.operation === 'embedMany') { const decoded = decodeEmbeddingRequest(request.payload); diff --git a/src/effect/ai/local-model-runtime.ts b/src/effect/ai/local-model-runtime.ts index 682efac9..cf52a9c9 100644 --- a/src/effect/ai/local-model-runtime.ts +++ b/src/effect/ai/local-model-runtime.ts @@ -41,7 +41,7 @@ export interface LocalGenerationRequest extends StructuredGenerationRequest { } export interface LocalModelRuntimeShape { - readonly diagnostics: () => Effect.Effect; + readonly diagnostics: Effect.Effect; readonly embedMany: ( request: LocalEmbeddingRequest, ) => Effect.Effect; @@ -61,13 +61,12 @@ export class LocalModelRuntime extends Context.Service( - engineLayer: Layer.Layer = nodeLlamaCppEngineLayer() as Layer.Layer< - LlamaCppEngine, - NativeRuntimeError, - R - >, -) { +export function localModelRuntimeLayer(): Layer.Layer; +export function localModelRuntimeLayer( + engineLayer: Layer.Layer, +): Layer.Layer; +export function localModelRuntimeLayer(engineLayer?: Layer.Layer) { + if (engineLayer === undefined) return localModelRuntimeLayer(nodeLlamaCppEngineLayer()); return Layer.fromBuild((_memoMap, scope) => Effect.gen(function* () { const inferencePermits = yield* Semaphore.make(1); @@ -81,7 +80,7 @@ export function localModelRuntimeLayer( Effect.Effect >(); return Context.make(LocalModelRuntime, { - diagnostics: () => engineContext.pipe(Effect.map(context => Context.get(context, LlamaCppEngine).diagnostics)), + diagnostics: engineContext.pipe(Effect.map(context => Context.get(context, LlamaCppEngine).diagnostics)), embedMany: request => inferencePermits.withPermit(embedManyNative(request, scope, engineContext, embeddingModels)), generate: request => inferencePermits.withPermit(generateNative(request, engineContext)), diff --git a/src/effect/ai/mcp.ts b/src/effect/ai/mcp.ts index c6e8f3a8..bd898d21 100644 --- a/src/effect/ai/mcp.ts +++ b/src/effect/ai/mcp.ts @@ -2,7 +2,7 @@ import * as BunStdio from '@effect/platform-bun/BunStdio'; import {Cause, Context, Effect, Layer, Logger, Option, Schema, Sink, Stdio} from 'effect'; import {McpSchema, McpServer} from 'effect/unstable/ai'; import {RpcMessage, RpcSerialization, RpcServer} from 'effect/unstable/rpc'; -import {fromPromiseError} from '../errors.js'; +import {applicationError, fromPromise} from '../errors.js'; import type {ApplicationServices} from '../runtime.js'; import {omitProductionLogPhaseRecorder, withProductionLogging} from '../production_log.js'; @@ -13,6 +13,7 @@ const MCP_PRODUCTION_LOG_WRITE_TIMEOUT_MILLISECONDS = 500; const EFFECT_RPC_CAUSE_MARKER = new TextEncoder().encode('"_tag":"Cause"'); const MCP_RESOURCE_ERROR_BRAND_KEY = 'threadnote.io/resource-read-error'; export const MCP_RESOURCE_ERROR_DATA = Object.freeze({[MCP_RESOURCE_ERROR_BRAND_KEY]: 1}); +export const MCP_RESOURCE_NOT_FOUND_ERROR_DATA = Object.freeze({[MCP_RESOURCE_ERROR_BRAND_KEY]: 2}); export const MCP_PROGRESS_HEARTBEAT_MILLISECONDS = 10_000; export const MCP_PROGRESS_MESSAGE_MAX_BYTES = 160; export const MCP_PROGRESS_METADATA_KEY = 'threadnote.io/progress'; @@ -123,7 +124,7 @@ type ResourceTemplateHandler = ( uri: string, ) => Effect.Effect< typeof McpSchema.ReadResourceResult.Type, - McpSchema.InternalError | McpSchema.InvalidParams | McpSchema.McpErrorBase, + McpSchema.InternalError | McpSchema.InvalidParams, ApplicationServices >; @@ -200,18 +201,9 @@ export class EffectMcpServerAdapter { annotations: Context.empty(), completions: {}, handle: uri => - // The negotiated Effect MCP revision accepts McpErrorBase on - // the wire, but addResourceTemplate's beta type only lists two - // concrete subclasses. Preserve the wider protocol error here. registration .handle(uri) - .pipe( - Effect.provideContext(applicationServices), - Effect.catchCause(mcpResourceFailureResult), - ) as Effect.Effect< - typeof McpSchema.ReadResourceResult.Type, - McpSchema.InternalError | McpSchema.InvalidParams - >, + .pipe(Effect.provideContext(applicationServices), Effect.catchCause(mcpResourceFailureResult)), routerPath: registration.definition.routerPath, template: new McpSchema.ResourceTemplate({ _meta: registration.definition.meta, @@ -714,17 +706,13 @@ function boundedMcpProgressMessage(value: string): string { export function mcpResourceFailureResult( cause: Cause.Cause, -): Effect.Effect { +): Effect.Effect { if (Cause.hasInterrupts(cause)) { - return Effect.failCause( - cause as Cause.Cause, - ); + return Effect.failCause(cause as Cause.Cause); } const error = Option.getOrUndefined(Cause.findErrorOption(cause)); if ( - (error instanceof McpSchema.InvalidParams || - error instanceof McpSchema.InternalError || - error instanceof McpSchema.McpErrorBase) && + (error instanceof McpSchema.InvalidParams || error instanceof McpSchema.InternalError) && hasMcpResourceErrorBrand(error) ) { return Effect.fail(error); @@ -790,13 +778,13 @@ function toolHandlerEffect( evaluate: () => ToolHandlerResult, applicationServices: Context.Context, ): Effect.Effect { - return Effect.try({try: evaluate, catch: normalizeError}).pipe( + return Effect.try({try: evaluate, catch: cause => applicationError('evaluate MCP tool handler', cause)}).pipe( Effect.flatMap(handled => { if (Effect.isEffect(handled)) { return handled.pipe(Effect.provideContext(applicationServices)); } if (isPromiseLike(handled)) { - return fromPromiseError(() => Promise.resolve(handled)); + return fromPromise('handle Effect AI MCP request', () => Promise.resolve(handled)); } return Effect.succeed(handled); }), @@ -807,10 +795,6 @@ function isPromiseLike(value: unknown): value is PromiseLike { return typeof value === 'object' && value !== null && 'then' in value && typeof value.then === 'function'; } -function normalizeError(cause: unknown): Error { - return cause instanceof Error ? cause : new Error(String(cause)); -} - const annotate = (schema: S, description?: string): S['Rebuild'] => schema.annotate(description ? {description} : {}); @@ -955,7 +939,7 @@ function unwrapEffectRpcMcpError(parsed: Record): Record { return Effect.gen(function* () { @@ -67,7 +67,7 @@ function resourceTooLarge(): McpSchema.InvalidParams { function mcpResourceReadError( error: ResourceStoreError | McpSchema.InternalError | McpSchema.InvalidParams, -): McpSchema.InternalError | McpSchema.InvalidParams | McpSchema.McpErrorBase { +): McpSchema.InternalError | McpSchema.InvalidParams { if (error instanceof McpSchema.InvalidParams || error instanceof McpSchema.InternalError) { return error; } @@ -83,9 +83,8 @@ function mcpResourceReadError( message: 'Threadnote resource is not readable in the active account.', }); case 'ResourceNotFound': - return new McpSchema.McpErrorBase({ - code: MCP_RESOURCE_NOT_FOUND_CODE, - data: MCP_RESOURCE_ERROR_DATA, + return new McpSchema.InvalidParams({ + data: MCP_RESOURCE_NOT_FOUND_ERROR_DATA, message: 'Threadnote resource was not found.', }); default: diff --git a/src/effect/ai/recall.ts b/src/effect/ai/recall.ts index fb322798..a89c6ae1 100644 --- a/src/effect/ai/recall.ts +++ b/src/effect/ai/recall.ts @@ -185,13 +185,19 @@ export const runEffectAiRecallExpansion = Effect.fn('RecallQueryExpander.run')(f expansionCache.set(fingerprint, cached); return cached; } - const rewrites = yield* expandRecallQueryEffect(input).pipe( - Effect.provide(recallQueryExpanderLayer(config)), - Effect.timeoutOrElse({ - duration: RECALL_EXPANSION_TIMEOUT_MILLISECONDS, - orElse: () => Effect.succeed([]), - }), - Effect.catch(() => Effect.succeed([])), + const rewrites = yield* Effect.scoped( + Layer.build(recallQueryExpanderLayer(config)).pipe( + Effect.flatMap(context => + expandRecallQueryEffect(input).pipe( + Effect.provide(context), + Effect.timeoutOrElse({ + duration: RECALL_EXPANSION_TIMEOUT_MILLISECONDS, + orElse: () => Effect.succeed([]), + }), + Effect.catch(() => Effect.succeed([])), + ), + ), + ), ); expansionCache.set(fingerprint, rewrites); while (expansionCache.size > MAX_RECALL_EXPANSION_CACHE_ENTRIES) { @@ -215,7 +221,6 @@ export const expandWeakRecallQueryEffect = Effect.fn('RecallQueryExpander.expand duration: RECALL_EXPANSION_TIMEOUT_MILLISECONDS, orElse: () => Effect.succeed(false), }), - Effect.catch(() => Effect.succeed(false)), ); if (!ready) return []; const config = resolved.configuration; @@ -368,8 +373,10 @@ const runEffectAiRecallSelection = Effect.fn('RecallCandidateSelector.run')(func selectionCache.set(fingerprint, cached); return cached; } - const selected = yield* selectRecallCandidatesEffect(input).pipe( - Effect.provide(recallCandidateSelectorLayer(config)), + const selected = yield* Effect.scoped( + Layer.build(recallCandidateSelectorLayer(config)).pipe( + Effect.flatMap(context => selectRecallCandidatesEffect(input).pipe(Effect.provide(context))), + ), ); selectionCache.set(fingerprint, selected); while (selectionCache.size > MAX_RECALL_EXPANSION_CACHE_ENTRIES) { diff --git a/src/effect/archive.ts b/src/effect/archive.ts index ddeafdf6..fb2d620f 100644 --- a/src/effect/archive.ts +++ b/src/effect/archive.ts @@ -1,5 +1,9 @@ import {Effect, Exit, FileSystem, Path, Scope, Stream} from 'effect'; +class ArchiveOperationError extends Error { + readonly _tag = 'ArchiveOperationError' as const; +} + const TAR_BLOCK_BYTES = 512; const MAX_COMPRESSED_ARCHIVE_BYTES = 1024 * 1024 * 1024; const MAX_EXPANDED_ARCHIVE_BYTES = 4 * 1024 * 1024 * 1024; @@ -36,7 +40,7 @@ export const extractGzipTar = Effect.fn('archive.extractGzipTar')(function* ( const archiveInfo = yield* fs.stat(archivePath); if (Number(archiveInfo.size) > MAX_COMPRESSED_ARCHIVE_BYTES) { return yield* Effect.fail( - new Error(`Release archive exceeds ${MAX_COMPRESSED_ARCHIVE_BYTES} compressed bytes.`), + new ArchiveOperationError(`Release archive exceeds ${MAX_COMPRESSED_ARCHIVE_BYTES} compressed bytes.`), ); } const limits: Required = { @@ -50,26 +54,33 @@ export const extractGzipTar = Effect.fn('archive.extractGzipTar')(function* ( const parser = createTarStreamParser(fs, path, parentScope, destination, limits); const decompressed = Stream.fromReadableStream({ evaluate: () => Bun.file(archivePath).stream().pipeThrough(new DecompressionStream('gzip')), - onError: cause => new Error(`Could not decompress ${archivePath}.`, {cause}), + onError: cause => new ArchiveOperationError(`Could not decompress ${archivePath}.`, {cause}), }); let decompressedBytes = 0; yield* decompressed.pipe( Stream.runForEach(chunk => Effect.gen(function* () { if (chunk.length > MAX_DECOMPRESSED_CHUNK_BYTES) { - return yield* Effect.fail(new Error(`Release archive emitted an oversized ${chunk.length}-byte chunk.`)); + return yield* Effect.fail( + new ArchiveOperationError(`Release archive emitted an oversized ${chunk.length}-byte chunk.`), + ); } decompressedBytes += chunk.length; if (decompressedBytes > limits.decompressedBytes) { return yield* Effect.fail( - new Error(`Release archive exceeds ${limits.decompressedBytes} decompressed tar bytes.`), + new ArchiveOperationError( + `Release archive exceeds ${limits.decompressedBytes} decompressed tar bytes.`, + ), ); } yield* parser.write(chunk); }), ), Effect.andThen(parser.complete()), - Effect.mapError(cause => new Error(`Could not extract ${archivePath}: ${archiveCauseMessage(cause)}`, {cause})), + Effect.mapError( + cause => + new ArchiveOperationError(`Could not extract ${archivePath}: ${archiveCauseMessage(cause)}`, {cause}), + ), ); }), ); @@ -109,11 +120,13 @@ function createTarStreamParser( while (buffer.length > 0) { if (ended) { if (!buffer.every(byte => byte === 0)) { - return yield* Effect.fail(new Error('Release archive contains data after its end marker.')); + return yield* Effect.fail(new ArchiveOperationError('Release archive contains data after its end marker.')); } endPaddingBytes += buffer.length; if (endPaddingBytes > limits.endPaddingBytes) { - return yield* Effect.fail(new Error(`Release archive exceeds ${limits.endPaddingBytes} end-padding bytes.`)); + return yield* Effect.fail( + new ArchiveOperationError(`Release archive exceeds ${limits.endPaddingBytes} end-padding bytes.`), + ); } buffer = new Uint8Array(); return; @@ -163,16 +176,20 @@ function createTarStreamParser( } entryCount += 1; if (entryCount > limits.entries) { - return yield* Effect.fail(new Error(`Release archive exceeds ${limits.entries} tar entries.`)); + return yield* Effect.fail(new ArchiveOperationError(`Release archive exceeds ${limits.entries} tar entries.`)); } const entry = yield* attemptArchiveParse(() => parseTarHeader(header)); expandedBytes += entry.size; if (expandedBytes > limits.expandedBytes) { - return yield* Effect.fail(new Error(`Release archive exceeds ${limits.expandedBytes} expanded bytes.`)); + return yield* Effect.fail( + new ArchiveOperationError(`Release archive exceeds ${limits.expandedBytes} expanded bytes.`), + ); } if (entry.type === 'globalPax' || entry.type === 'longPath' || entry.type === 'pax') { if (entry.size > MAX_TAR_METADATA_BYTES) { - return yield* Effect.fail(new Error(`Release archive tar metadata exceeds ${MAX_TAR_METADATA_BYTES} bytes.`)); + return yield* Effect.fail( + new ArchiveOperationError(`Release archive tar metadata exceeds ${MAX_TAR_METADATA_BYTES} bytes.`), + ); } file = { chunks: [], @@ -198,7 +215,9 @@ function createTarStreamParser( : yield* attemptArchiveParse(() => parsePaxSize(effectivePax.size!)); expandedBytes += effectiveSize - entry.size; if (expandedBytes > limits.expandedBytes) { - return yield* Effect.fail(new Error(`Release archive exceeds ${limits.expandedBytes} expanded bytes.`)); + return yield* Effect.fail( + new ArchiveOperationError(`Release archive exceeds ${limits.expandedBytes} expanded bytes.`), + ); } nextLongPath = undefined; nextPax = undefined; @@ -238,10 +257,12 @@ function createTarStreamParser( const complete = Effect.fn('archive.tarParser.complete')(function* () { if (!ended || file || paddingRemaining !== 0 || buffer.length !== 0) { - return yield* Effect.fail(new Error('Release archive ended before a complete tar end marker.')); + return yield* Effect.fail(new ArchiveOperationError('Release archive ended before a complete tar end marker.')); } if (nextLongPath !== undefined || nextPax !== undefined) { - return yield* Effect.fail(new Error('Release archive ended after metadata without a target entry.')); + return yield* Effect.fail( + new ArchiveOperationError('Release archive ended after metadata without a target entry.'), + ); } }); @@ -252,14 +273,17 @@ function validateExtractionLimits(limits: Required) { return Effect.forEach(Object.entries(limits), ([name, value]) => Number.isSafeInteger(value) && value > 0 ? Effect.void - : Effect.fail(new Error(`Archive extraction limit ${name} must be a positive safe integer.`)), + : Effect.fail(new ArchiveOperationError(`Archive extraction limit ${name} must be a positive safe integer.`)), ).pipe(Effect.asVoid); } function attemptArchiveParse(parse: () => A) { return Effect.try({ try: parse, - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: cause => + cause instanceof ArchiveOperationError + ? cause + : new ArchiveOperationError(cause instanceof Error ? cause.message : String(cause), {cause}), }); } @@ -270,13 +294,14 @@ function archiveCauseMessage(cause: unknown): string { function parseTarHeader(header: Uint8Array): TarEntry { const expectedChecksum = parseTarNumber(header.slice(148, 156), 'checksum'); const checksum = header.reduce((total, byte, index) => total + (index >= 148 && index < 156 ? 0x20 : byte), 0); - if (checksum !== expectedChecksum) throw new Error('Release archive contains an invalid tar header checksum.'); + if (checksum !== expectedChecksum) + throw new ArchiveOperationError('Release archive contains an invalid tar header checksum.'); const name = tarText(header.slice(0, 100)); const prefix = tarText(header.slice(345, 500)); const entryPath = prefix ? `${prefix}/${name}` : name; const type = String.fromCharCode(header[156] ?? 0); if (type !== '\0' && type !== '0' && type !== '5' && type !== 'g' && type !== 'L' && type !== 'x') { - throw new Error(`Release archive contains unsupported tar entry type ${JSON.stringify(type)}.`); + throw new ArchiveOperationError(`Release archive contains unsupported tar entry type ${JSON.stringify(type)}.`); } return { mode: parseTarNumber(header.slice(100, 108), 'mode') & 0o777, @@ -300,17 +325,18 @@ function parsePaxMetadata(bytes: Uint8Array): Readonly bytes.length || bytes[end - 1] !== 0x0a) { - throw new Error('Release archive contains a truncated PAX metadata record.'); + throw new ArchiveOperationError('Release archive contains a truncated PAX metadata record.'); } const record = new TextDecoder().decode(bytes.slice(separator + 1, end - 1)); const equals = record.indexOf('='); - if (equals <= 0) throw new Error('Release archive contains malformed PAX metadata.'); + if (equals <= 0) throw new ArchiveOperationError('Release archive contains malformed PAX metadata.'); metadata[record.slice(0, equals)] = record.slice(equals + 1); offset = end; } @@ -330,9 +356,11 @@ function mergePaxMetadata( } function parsePaxSize(value: string): number { - if (!/^(?:0|[1-9][0-9]*)$/.test(value)) throw new Error('Release archive contains an invalid PAX size.'); + if (!/^(?:0|[1-9][0-9]*)$/.test(value)) + throw new ArchiveOperationError('Release archive contains an invalid PAX size.'); const size = Number.parseInt(value, 10); - if (!Number.isSafeInteger(size) || size < 0) throw new Error('Release archive contains an invalid PAX size.'); + if (!Number.isSafeInteger(size) || size < 0) + throw new ArchiveOperationError('Release archive contains an invalid PAX size.'); return size; } @@ -348,16 +376,16 @@ function safeArchivePath(path: Path.Path, destination: string, entry: string): s .replace(/^\.\/+/, '') .replace(/\/+$/, ''); if (!normalized || normalized.startsWith('/') || /^[A-Za-z]:\//.test(normalized)) { - throw new Error(`Release archive contains an unsafe path: ${entry}`); + throw new ArchiveOperationError(`Release archive contains an unsafe path: ${entry}`); } const segments = normalized.split('/'); if (segments.some(segment => !segment || segment === '.' || segment === '..')) { - throw new Error(`Release archive contains an unsafe path: ${entry}`); + throw new ArchiveOperationError(`Release archive contains an unsafe path: ${entry}`); } const root = path.resolve(destination); const resolved = path.resolve(root, ...segments); if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) { - throw new Error(`Release archive path escapes its destination: ${entry}`); + throw new ArchiveOperationError(`Release archive path escapes its destination: ${entry}`); } return resolved; } @@ -368,9 +396,10 @@ function isArchiveRootPath(entry: string): boolean { function parseTarNumber(bytes: Uint8Array, field: string): number { const text = tarText(bytes).trim(); - if (!/^[0-7]+$/.test(text)) throw new Error(`Release archive has an invalid tar ${field}.`); + if (!/^[0-7]+$/.test(text)) throw new ArchiveOperationError(`Release archive has an invalid tar ${field}.`); const value = Number.parseInt(text, 8); - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Release archive has an invalid tar ${field}.`); + if (!Number.isSafeInteger(value) || value < 0) + throw new ArchiveOperationError(`Release archive has an invalid tar ${field}.`); return value; } diff --git a/src/effect/cli_output.ts b/src/effect/cli_output.ts index aa53260d..a6651bd0 100644 --- a/src/effect/cli_output.ts +++ b/src/effect/cli_output.ts @@ -1,5 +1,9 @@ import {Console, Context, Effect, Layer, Logger} from 'effect'; +class CliOutputError extends Error { + readonly _tag = 'CliOutputError' as const; +} + export interface CliOutputShape { readonly drain: Effect.Effect; readonly enqueueError: (output: string) => void; @@ -13,7 +17,7 @@ export function makeFinalCliOutput(write: (output: string) => Promise) { return Effect.fn('cliOutput.writeFinal')(function* (output: string) { yield* Effect.tryPromise({ try: () => write(output), - catch: cause => new Error('Failed to write complete Threadnote CLI output.', {cause}), + catch: cause => new CliOutputError('Failed to write complete Threadnote CLI output.', {cause}), }); }); } @@ -74,13 +78,13 @@ export class CliOutput extends Context.Service()('thr return CliOutput.of({ drain: Effect.tryPromise({ try: () => Promise.all([stdout.drain(), stderr.drain()]).then(() => undefined), - catch: cause => new Error('Failed to drain Threadnote CLI output.', {cause}), + catch: cause => new CliOutputError('Failed to drain Threadnote CLI output.', {cause}), }), enqueueError: stderr.enqueue, enqueueOutput: stdout.enqueue, flush: Effect.tryPromise({ try: () => Promise.all([stdout.flush(), stderr.flush()]).then(() => undefined), - catch: cause => new Error('Failed to flush Threadnote CLI output.', {cause}), + catch: cause => new CliOutputError('Failed to flush Threadnote CLI output.', {cause}), }), writeError: makeFinalCliOutput(stderr.write), writeFinal: makeFinalCliOutput(stdout.write), diff --git a/src/effect/errors.ts b/src/effect/errors.ts index 5d58d573..d3ff3460 100644 --- a/src/effect/errors.ts +++ b/src/effect/errors.ts @@ -22,11 +22,5 @@ export const fromPromiseInterruptible = ( onError: (cause: unknown) => E, ) => Effect.tryPromise({try: evaluate, catch: onError}); -export const fromPromiseError = (evaluate: () => PromiseLike) => - Effect.tryPromise({ - try: evaluate, - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), - }); - export const fromSync = (operation: string, evaluate: () => A) => Effect.try({try: evaluate, catch: cause => applicationError(operation, cause)}); diff --git a/src/effect/local-ai.ts b/src/effect/local-ai.ts index 7e3618b3..fd5b3ecd 100644 --- a/src/effect/local-ai.ts +++ b/src/effect/local-ai.ts @@ -7,6 +7,10 @@ import {clearLocalModelSelection, readModelSelection, selectLocalModel} from '.. import {LocalModelStore} from '../models/store.js'; import type {RuntimeConfig} from '../types.js'; +class LocalAiOperationError extends Error { + readonly _tag = 'LocalAiOperationError' as const; +} + const COMPATIBILITY_MODEL_ID = 'gemma-4-e4b-it-q4'; const compatibilityModel = BUILTIN_MODEL_MANIFESTS.find(model => model.id === COMPATIBILITY_MODEL_ID)!; @@ -61,7 +65,7 @@ export const runLocalAiInstall = Effect.fn('localAi.compat.install')(function* ( ) { if (options.modelPath) { return yield* Effect.fail( - new Error( + new LocalAiOperationError( '`threadnote local-ai --model-path` was removed in 4.0 because unmanaged model files bypass the signed catalog. Use `threadnote models install`.', ), ); @@ -78,7 +82,9 @@ export const runLocalAiModelSwitch = Effect.fn('localAi.compat.switch')(function ) { if (!options.model) { return yield* Effect.fail( - new Error('Specify `--model `, or use `threadnote models list` and `threadnote models select generation`.'), + new LocalAiOperationError( + 'Specify `--model `, or use `threadnote models list` and `threadnote models select generation`.', + ), ); } yield* Console.warn('`threadnote local-ai model switch` is deprecated; use `threadnote models select generation`.'); @@ -92,7 +98,7 @@ export const runLocalAiEnable = Effect.fn('localAi.compat.enable')(function* ( const models = yield* installedGenerationModels(config); if (models.length === 0) { return yield* Effect.fail( - new Error('No generation model is installed. Run `threadnote models install ` first.'), + new LocalAiOperationError('No generation model is installed. Run `threadnote models install ` first.'), ); } const selected = models[0]!; @@ -213,7 +219,7 @@ export function parseLocalAiSettings(value: unknown): LocalAiSettings { typeof (value as Partial).model !== 'string' || typeof (value as Partial).modelPath !== 'string' ) { - throw new Error('Legacy local-ai server settings are not valid Threadnote 4 settings.'); + throw new LocalAiOperationError('Legacy local-ai server settings are not valid Threadnote 4 settings.'); } return value as LocalAiSettings; } @@ -237,7 +243,7 @@ function requireSelectedGeneration(config: LocalAiRuntimeConfig) { selected ? Effect.succeed(selected) : Effect.fail( - new Error( + new LocalAiOperationError( 'No generation model is selected. Use `threadnote models install` and `threadnote models select generation`.', ), ), diff --git a/src/effect/memory_lock.ts b/src/effect/memory_lock.ts index 67fbf31f..aad637ab 100644 --- a/src/effect/memory_lock.ts +++ b/src/effect/memory_lock.ts @@ -1,6 +1,7 @@ -import {Effect, FileSystem, Path} from 'effect'; +import {Crypto, Effect, FileSystem, Path} from 'effect'; import {sha256Hex} from './digest.js'; import {withExclusiveFileLock} from './file_lock.js'; +import {SystemInfo} from './system.js'; const MEMORY_LOCK_STALE_MILLISECONDS = 5 * 60 * 1_000; const MEMORY_LOCK_RETRY_MILLISECONDS = 25; @@ -29,12 +30,11 @@ export function withMemoryUriLocks( Effect.map(digest => pathService.join(agentContextHome, 'threadnote', 'memory-locks', `${digest}.lock`)), ), ); - return yield* lockPaths - .sort() - .reduceRight>( - (protectedEffect, lockPath) => - withExclusiveFileLock(fs, lockPath, MEMORY_LOCK_OPTIONS, protectedEffect) as Effect.Effect, - effect, - ); + const sortedLockPaths = lockPaths.sort(); + const protect = (index: number): Effect.Effect => + index >= sortedLockPaths.length + ? effect + : withExclusiveFileLock(fs, sortedLockPaths[index]!, MEMORY_LOCK_OPTIONS, protect(index + 1)); + return yield* protect(0); }); } diff --git a/src/effect/resource-store.ts b/src/effect/resource-store.ts index 4df36355..23481997 100644 --- a/src/effect/resource-store.ts +++ b/src/effect/resource-store.ts @@ -176,16 +176,13 @@ export class ResourceStore extends Context.Service(effect: Effect.Effect) => - effect.pipe( - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(Path.Path, path), - Effect.provideService(SystemInfo, system), - ) as Effect.Effect>; + const lockServices = yield* Effect.context(); + const provideLockServices = ( + effect: Effect.Effect, + ): Effect.Effect> => + effect.pipe(Effect.provide(lockServices)); const operation = createResourceStoreOperations(fs, path, provideLockServices, options); return ResourceStore.of(operation); }), @@ -217,11 +214,7 @@ function createResourceStoreOperations( ); const invalidateRecallBestEffort = (location: ResourceStoreLocation, invalidatedUris: readonly string[]) => invalidateRecall(location, invalidatedUris).pipe(Effect.catchCause(() => Effect.void)); - const withLock = ( - location: ResourceStoreLocation, - id: ResourceId, - effect: Effect.Effect, - ): Effect.Effect> => { + const withLock = (location: ResourceStoreLocation, id: ResourceId, effect: Effect.Effect) => { const lockPath = resourceAccountMutationLockPath(path, location.home, location.account); const event = {account: location.account, lockPath, uri: id.canonicalUri}; const lockEffect = withExclusiveFileLock( @@ -264,7 +257,7 @@ function createResourceStoreOperations( onSuccess: value => Effect.succeed(value), }), ), - ) as Effect.Effect>; + ); }; const removeResource = (location: ResourceStoreLocation, uri: string, options?: {readonly recursive?: boolean}) => Effect.gen(function* () { diff --git a/src/effect/system.ts b/src/effect/system.ts index 6954991b..af515680 100644 --- a/src/effect/system.ts +++ b/src/effect/system.ts @@ -2,12 +2,25 @@ import {Context, Deferred, Effect, Exit, Layer, Ref} from 'effect'; import {effectiveLinuxMemoryBytes, linuxCgroupMemoryFiles} from './linux_cgroup.js'; import {readWindowsHardwareInfo, readWindowsProcessStartIdentity} from './windows_system.js'; +class SystemOperationError extends Error { + readonly _tag = 'SystemOperationError' as const; +} + +function systemOperationError(cause: unknown): SystemOperationError { + return cause instanceof SystemOperationError + ? cause + : new SystemOperationError(cause instanceof Error ? cause.message : String(cause), {cause}); +} + export interface PlatformPathShape { readonly basename: (path: string) => string; readonly dirname: (path: string) => string; readonly isAbsolute: (path: string) => boolean; readonly join: (...paths: readonly string[]) => string; readonly normalize: (path: string) => string; + readonly relative: (from: string, to: string) => string; + readonly resolve: (...paths: readonly string[]) => string; + readonly sep: string; } interface NativeFileSystemPromisesShape { @@ -16,6 +29,7 @@ interface NativeFileSystemPromisesShape { path: string, options: {readonly bufferSize: number; readonly encoding: 'buffer' | 'utf8'}, ) => Promise; + readonly stat: (path: string, options: {readonly bigint: true}) => Promise; readonly statfs?: (path: string, options: {readonly bigint: true}) => Promise; } @@ -24,6 +38,12 @@ interface NativePathModuleShape { readonly win32: PlatformPathShape; } +interface NativeOperatingSystemModuleShape { + readonly cpus: () => readonly {readonly model: string}[]; + readonly release: () => string; + readonly totalmem: () => number; +} + export interface RuntimeBigIntStats { readonly ctimeNs: bigint; readonly dev: bigint; @@ -32,6 +52,7 @@ export interface RuntimeBigIntStats { readonly mtimeNs: bigint; readonly size: bigint; readonly isDirectory: () => boolean; + readonly isFile: () => boolean; readonly isSymbolicLink: () => boolean; } @@ -56,6 +77,8 @@ export interface RuntimeTextDirectoryNamePage { /** Host facts and Bun's Node-compatible structural adapters stay inside SystemInfo's runtime boundary. */ export const runtimeArchitecture = process.arch; export const runtimePlatform = process.platform; +const nativeOperatingSystemModule = process.getBuiltinModule('os') as NativeOperatingSystemModuleShape; +export const runtimeOperatingSystemRelease = nativeOperatingSystemModule.release(); const nativeFileSystemPromises = (process.getBuiltinModule('fs') as {readonly promises: NativeFileSystemPromisesShape}) .promises; const nativePathModule = process.getBuiltinModule('path') as NativePathModuleShape; @@ -64,13 +87,33 @@ export function platformPathFor(platform: NodeJS.Platform): PlatformPathShape { return platform === 'win32' ? nativePathModule.win32 : nativePathModule.posix; } +/** Exact host facts retained by same-machine benchmark provenance. */ +export function runtimeHostHardwareInfo(): { + readonly cpuModel: string; + readonly logicalCpuCount: number; + readonly memoryBytes: number; +} { + const processors = nativeOperatingSystemModule.cpus(); + return { + cpuModel: processors[0]?.model ?? 'unknown', + logicalCpuCount: processors.length, + memoryBytes: nativeOperatingSystemModule.totalmem(), + }; +} + export function runtimeLstat(path: string): Promise { return nativeFileSystemPromises.lstat(path, {bigint: true}); } +/** Follows links while retaining exact device/inode identity beyond JavaScript's safe-integer range. */ +export function runtimeStat(path: string): Promise { + return nativeFileSystemPromises.stat(path, {bigint: true}); +} + /** Raw POSIX directory names stay bytes; enumeration stops immediately after the first over-limit entry. */ export async function runtimeDirectoryNamePage(path: string, entryLimit: number): Promise { - if (!Number.isSafeInteger(entryLimit) || entryLimit < 0) throw new Error('Runtime directory entry limit is invalid.'); + if (!Number.isSafeInteger(entryLimit) || entryLimit < 0) + throw new SystemOperationError('Runtime directory entry limit is invalid.'); const directory = await nativeFileSystemPromises.opendir(path, { bufferSize: 32, encoding: runtimePlatform === 'win32' ? 'utf8' : 'buffer', @@ -99,7 +142,7 @@ export function runtimeTextDirectoryNamePage( ): Effect.Effect { return Effect.tryPromise({ try: () => runtimeDirectoryNamePage(path, entryLimit), - catch: cause => cause, + catch: systemOperationError, }).pipe( Effect.flatMap(page => Effect.try({ @@ -107,7 +150,7 @@ export function runtimeTextDirectoryNamePage( const decoder = new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}); return {names: page.names.map(name => decoder.decode(name)), overflow: page.overflow}; }, - catch: cause => cause, + catch: systemOperationError, }), ), ); @@ -122,7 +165,7 @@ export interface SystemInfoShape { readonly environment: () => NodeJS.ProcessEnv; readonly executablePath: string; readonly homeDirectory: string; - readonly hardwareInfo: () => Effect.Effect; + readonly hardwareInfo: Effect.Effect; readonly isProcessRunning: (processId: number) => boolean; readonly memoryUsage: () => { readonly external: number; @@ -243,7 +286,7 @@ export class SystemInfo extends Context.Service()(' currentDirectory: () => process.cwd(), environment: () => process.env, executablePath: process.execPath, - hardwareInfo: () => readSystemHardwareInfo(runtimePlatform, process.env), + hardwareInfo: readSystemHardwareInfo(runtimePlatform, process.env), homeDirectory, isProcessRunning: processId => { try { @@ -423,7 +466,7 @@ function nativeStatfs(path: string) { } return Effect.tryPromise({ try: () => nativeFileSystemPromises.statfs!(path, {bigint: true}), - catch: cause => cause, + catch: systemOperationError, }); } @@ -459,7 +502,7 @@ export function legacyAvailableDiskBytes( stdin: 'ignore', stdout: 'pipe', }), - catch: cause => cause, + catch: systemOperationError, }), child => Effect.tryPromise({ @@ -469,7 +512,7 @@ export function legacyAvailableDiskBytes( const text = output.trim(); return platform === 'win32' ? parseWindowsAvailableDiskBytes(text) : parsePosixAvailableDiskBytes(text); }, - catch: cause => cause, + catch: systemOperationError, }), child => Effect.sync(() => { @@ -489,9 +532,10 @@ const defaultDiskCapacityProbeAdapters: DiskCapacityProbeAdapters = { }; function isNativeStatfsUnavailable(cause: unknown): boolean { - if (cause instanceof NativeStatfsUnavailableError) return true; - if (typeof cause !== 'object' || cause === null || !('code' in cause)) return false; - const code = (cause as {readonly code?: unknown}).code; + const underlying = cause instanceof SystemOperationError ? cause.cause : cause; + if (underlying instanceof NativeStatfsUnavailableError) return true; + if (typeof underlying !== 'object' || underlying === null || !('code' in underlying)) return false; + const code = (underlying as {readonly code?: unknown}).code; return typeof code === 'string' && NATIVE_STATFS_UNAVAILABLE_CODES.has(code); } @@ -531,14 +575,14 @@ function readSystemHardwareInfo(platform: NodeJS.Platform, environment: NodeJS.P const cpuModel = /^(?:model name|Hardware)\s*:\s*(.+)$/m.exec(cpuInfo)?.[1]?.trim(); const memoryKibibytes = Number(/^MemTotal:\s+(\d+)\s+kB$/m.exec(memoryInfo)?.[1]); if (!cpuModel || !Number.isSafeInteger(memoryKibibytes) || memoryKibibytes <= 0) { - throw new Error('Linux hardware metadata is incomplete.'); + throw new SystemOperationError('Linux hardware metadata is incomplete.'); } const memoryBytes = memoryKibibytes * KIBIBYTE_BYTES; const effectiveMemoryBytes = await readLinuxEffectiveMemoryBytes(memoryBytes); const operatingSystem = spawnText(['uname', '-sr'], environment); return {cpuModel, effectiveMemoryBytes, memoryBytes, operatingSystem}; }, - catch: cause => new Error('Could not read Linux hardware metadata.', {cause}), + catch: cause => new SystemOperationError('Could not read Linux hardware metadata.', {cause}), }); } if (platform === 'darwin') { @@ -548,17 +592,17 @@ function readSystemHardwareInfo(platform: NodeJS.Platform, environment: NodeJS.P const memoryBytes = Number(spawnText(['sysctl', '-n', 'hw.memsize'], environment)); const version = spawnText(['sw_vers', '-productVersion'], environment); if (!Number.isSafeInteger(memoryBytes) || memoryBytes <= 0) { - throw new Error('macOS memory metadata is invalid.'); + throw new SystemOperationError('macOS memory metadata is invalid.'); } return {cpuModel, effectiveMemoryBytes: memoryBytes, memoryBytes, operatingSystem: `macOS ${version}`}; }, - catch: cause => new Error('Could not read macOS hardware metadata.', {cause}), + catch: cause => new SystemOperationError('Could not read macOS hardware metadata.', {cause}), }); } if (platform === 'win32') { return readWindowsHardwareInfo(environment); } - return Effect.fail(new Error(`Hardware metadata is not supported on ${platform}.`)); + return Effect.fail(new SystemOperationError(`Hardware metadata is not supported on ${platform}.`)); } async function readLinuxEffectiveMemoryBytes(physicalMemoryBytes: number): Promise { @@ -589,10 +633,10 @@ function spawnText(command: readonly string[], environment: NodeJS.ProcessEnv): timeout: DISK_QUERY_TIMEOUT_MS, }); if (result.exitCode !== 0) { - throw new Error(`${command[0]} exited with ${result.exitCode}: ${result.stderr.toString().trim()}`); + throw new SystemOperationError(`${command[0]} exited with ${result.exitCode}: ${result.stderr.toString().trim()}`); } const output = result.stdout.toString().trim(); - if (!output) throw new Error(`${command[0]} returned no hardware metadata.`); + if (!output) throw new SystemOperationError(`${command[0]} returned no hardware metadata.`); return output; } @@ -663,7 +707,7 @@ function readDarwinProcessStartIdentity( stdin: 'ignore', stdout: 'pipe', }), - catch: cause => cause, + catch: systemOperationError, }), child => Effect.tryPromise({ @@ -671,7 +715,7 @@ function readDarwinProcessStartIdentity( const [exitCode, output] = await Promise.all([child.exited, new Response(child.stdout).text()]); return exitCode === 0 ? parseOutput(output) : undefined; }, - catch: cause => cause, + catch: systemOperationError, }), child => Effect.sync(() => { @@ -728,7 +772,7 @@ export function resolveHomeDirectory(environment: NodeJS.ProcessEnv, platform: N const windowsHome = userProfile ?? (homeDrive && homePath ? `${homeDrive}${homePath}` : undefined); const resolved = platform === 'win32' ? (windowsHome ?? home) : (home ?? windowsHome); if (!resolved) { - throw new Error('Could not determine the current user home directory from the environment.'); + throw new SystemOperationError('Could not determine the current user home directory from the environment.'); } return resolved; } diff --git a/src/effect/windows_system.ts b/src/effect/windows_system.ts index 24013aa6..88d9e922 100644 --- a/src/effect/windows_system.ts +++ b/src/effect/windows_system.ts @@ -1,6 +1,10 @@ import {dlopen} from 'bun:ffi'; import {Effect} from 'effect'; +class WindowsSystemError extends Error { + readonly _tag = 'WindowsSystemError' as const; +} + const PROCESS_QUERY_LIMITED_INFORMATION = 0x1000; const DOTNET_TICKS_AT_WINDOWS_FILE_TIME_EPOCH = 504_911_232_000_000_000n; const MEMORY_STATUS_BYTES = 64; @@ -37,14 +41,14 @@ export function readWindowsHardwareInfo(environment: NodeJS.ProcessEnv) { const memoryView = new DataView(memoryStatus.buffer); memoryView.setUint32(0, MEMORY_STATUS_BYTES, true); if (kernel.symbols.GlobalMemoryStatusEx(memoryStatus) === 0) { - throw new Error('GlobalMemoryStatusEx failed.'); + throw new WindowsSystemError('GlobalMemoryStatusEx failed.'); } const versionInfo = new Uint8Array(WINDOWS_VERSION_INFO_BYTES); const versionView = new DataView(versionInfo.buffer); versionView.setUint32(0, WINDOWS_VERSION_INFO_BYTES, true); if (native.symbols.RtlGetVersion(versionInfo) !== 0) { - throw new Error('RtlGetVersion failed.'); + throw new WindowsSystemError('RtlGetVersion failed.'); } const memoryBytes = Number(memoryView.getBigUint64(MEMORY_STATUS_TOTAL_PHYSICAL_OFFSET, true)); @@ -55,7 +59,7 @@ export function readWindowsHardwareInfo(environment: NodeJS.ProcessEnv) { true, )}.${versionView.getUint32(WINDOWS_VERSION_BUILD_OFFSET, true)}`; if (!Number.isSafeInteger(memoryBytes) || memoryBytes <= 0) { - throw new Error('Windows memory metadata is invalid.'); + throw new WindowsSystemError('Windows memory metadata is invalid.'); } return { cpuModel, @@ -68,7 +72,7 @@ export function readWindowsHardwareInfo(environment: NodeJS.ProcessEnv) { kernel.close(); } }, - catch: cause => new Error('Could not read native Windows hardware metadata.', {cause}), + catch: cause => new WindowsSystemError('Could not read native Windows hardware metadata.', {cause}), }); } diff --git a/src/hooks.ts b/src/hooks.ts index 02199c12..5ab67cc3 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -261,11 +261,14 @@ const captureTraceContext = Effect.fn('hooks.captureTraceContext')(() => Effect.gen(function* () { const payload = yield* readHookPayload(); if (!payload) { - return {}; + return {} satisfies TraceContext; } const rawTrace = payload.transcriptPath ? yield* distillTrace(payload.transcriptPath) : undefined; - return {sessionId: payload.sessionId, trace: rawTrace ? scrubTrace(rawTrace) : undefined}; - }).pipe(Effect.catch(() => Effect.succeed({} as TraceContext))), + return { + sessionId: payload.sessionId, + trace: rawTrace ? scrubTrace(rawTrace) : undefined, + } satisfies TraceContext; + }), ); /** Redacts soft leaks; drops the trace on a hard credential blocker. */ diff --git a/src/installations.ts b/src/installations.ts index dc58f78f..26845a02 100644 --- a/src/installations.ts +++ b/src/installations.ts @@ -13,6 +13,10 @@ import { } from './standalone_process_lease.js'; import {compareVersions} from './utils.js'; +class InstallationOperationError extends Error { + readonly _tag = 'InstallationOperationError' as const; +} + const ACTIVE_RELEASE_FILE = 'active-release.json'; const ACTIVE_RELEASE_BACKUP_FILE = 'active-release.previous.json'; const ACTIVE_RELEASE_JOURNAL_FILE = 'active-release.promotion.json'; @@ -120,12 +124,16 @@ export const promoteStandaloneReleaseDirectory = Effect.fn('installations.promot const versionsRoot = path.dirname(resolvedReleaseRoot); const releaseName = path.basename(resolvedReleaseRoot); if (!RELEASE_VERSION_PATTERN.test(releaseName)) { - return yield* Effect.fail(new Error(`Cannot promote an invalid standalone release path: ${releaseRoot}`)); + return yield* Effect.fail( + new InstallationOperationError(`Cannot promote an invalid standalone release path: ${releaseRoot}`), + ); } const resolvedStagedRoot = path.resolve(stagedRoot); if (!isStandaloneStagingPath(path, versionsRoot, releaseName, resolvedStagedRoot)) { return yield* Effect.fail( - new Error(`Standalone release staging path is not recognized within ${versionsRoot}: ${stagedRoot}`), + new InstallationOperationError( + `Standalone release staging path is not recognized within ${versionsRoot}: ${stagedRoot}`, + ), ); } yield* recoverStandaloneReleasePromotion(fs, path, resolvedReleaseRoot); @@ -400,7 +408,9 @@ const readActiveReleasePromotion = Effect.fn('installations.readActiveReleasePro path.dirname(value.temporaryPath) !== root || !/^\.active-release\.[0-9]+-[0-9a-f-]+\.next\.json$/i.test(path.basename(value.temporaryPath)) ) { - return yield* Effect.fail(new Error(`Active release promotion journal is invalid: ${journalPath}`)); + return yield* Effect.fail( + new InstallationOperationError(`Active release promotion journal is invalid: ${journalPath}`), + ); } return { activePath, @@ -442,7 +452,9 @@ const readReleaseDirectoryPromotion = Effect.fn('installations.readReleaseDirect rebasedStagedRoot === undefined || !isStandaloneStagingPath(path, versionsRoot, path.basename(releaseRoot), rebasedStagedRoot) ) { - return yield* Effect.fail(new Error(`Standalone release promotion journal is invalid: ${journalPath}`)); + return yield* Effect.fail( + new InstallationOperationError(`Standalone release promotion journal is invalid: ${journalPath}`), + ); } return { backupRoot, @@ -461,11 +473,11 @@ const parsePromotionJournal = Effect.fn('installations.parsePromotionJournal')(f try: () => { const value = JSON.parse(content) as unknown; if (typeof value !== 'object' || value === null || Array.isArray(value)) { - throw new Error('expected a JSON object'); + throw new InstallationOperationError('expected a JSON object'); } return value as Record; }, - catch: cause => new Error(`Could not parse promotion journal ${journalPath}.`, {cause}), + catch: cause => new InstallationOperationError(`Could not parse promotion journal ${journalPath}.`, {cause}), }); }); diff --git a/src/lifecycle.ts b/src/lifecycle.ts index f45091f8..9eec89b5 100644 --- a/src/lifecycle.ts +++ b/src/lifecycle.ts @@ -69,6 +69,10 @@ import { toolRoot, } from './utils.js'; +class LifecycleOperationError extends Error { + readonly _tag = 'LifecycleOperationError' as const; +} + const LAYOUT_RECEIPT = 'layout.json'; type UserAgentInstructionTarget = (typeof USER_AGENT_INSTRUCTION_TARGETS)[number]; interface RunInstallOptions extends InstallOptions { @@ -324,7 +328,7 @@ export const runRepair = Effect.fn('lifecycle.repair')(function* (config: Runtim `Rebuilt recall indexes for ${documentCount} document(s) and ${vectors.chunkCount} vector chunk(s).`, ), ), - Effect.mapError(cause => new Error(`Recall index repair failed: ${errorMessage(cause)}`)), + Effect.mapError(cause => new LifecycleOperationError(`Recall index repair failed: ${errorMessage(cause)}`)), ); } else { yield* Console.log('Would validate and rebuild the derived lexical and vector recall indexes.'); @@ -349,7 +353,9 @@ export const runRepair = Effect.fn('lifecycle.repair')(function* (config: Runtim Effect.andThen(runDoctor(config, {codeGraphCheck: completion.doctorCheck, dryRun, strict: false})), ), {migrateSchema: true, mode: options.deep === true ? 'deep' : 'quick'}, - ).pipe(Effect.mapError(cause => new Error(`Native code graph repair failed: ${errorMessage(cause)}`))); + ).pipe( + Effect.mapError(cause => new LifecycleOperationError(`Native code graph repair failed: ${errorMessage(cause)}`)), + ); if (options.postUpdate !== false) { yield* maybeRunPostUpdateAfterRepair(config, {dryRun}); } @@ -380,7 +386,7 @@ const maintainRecallIndexes = Effect.fn('lifecycle.maintainRecallIndexes')(funct }); return {documentCount: index.candidates.length, vectors}; }), - progress => progress.stop(), + progress => progress.stop, ); }); @@ -480,7 +486,9 @@ export const runUninstall = Effect.fn('lifecycle.uninstall')(function* ( ) { const dryRun = options.dryRun === true; if (options.eraseMemories === true && options.preserveMemories === true) { - return yield* Effect.fail(new Error('Use either --erase-memories or --preserve-memories, not both.')); + return yield* Effect.fail( + new LifecycleOperationError('Use either --erase-memories or --preserve-memories, not both.'), + ); } yield* removeMcpConfigs(options.mcp ?? 'available', dryRun); yield* removeMcpSnippets(config, dryRun); diff --git a/src/manager.ts b/src/manager.ts index bdcb56f7..7607aff1 100644 --- a/src/manager.ts +++ b/src/manager.ts @@ -1,5 +1,5 @@ import * as BunHttpServer from '@effect/platform-bun/BunHttpServer'; -import {Console, Crypto, Effect, Encoding, FileSystem, Option, Path, Result} from 'effect'; +import {Console, Crypto, Effect, Encoding, FileSystem, Layer, Option, Path, Ref, Result, Scope} from 'effect'; import * as HttpServer from 'effect/unstable/http/HttpServer'; import * as HttpServerRequest from 'effect/unstable/http/HttpServerRequest'; import * as HttpServerResponse from 'effect/unstable/http/HttpServerResponse'; @@ -56,7 +56,31 @@ import {collectDoctorChecks, runRepair, runStart} from './lifecycle.js'; import {runSeed, runSeedSkills} from './seeding.js'; import {currentPackageVersion, fetchLatestVersion, releaseSource} from './update.js'; import {selectUpdateChannel} from './update_channel.js'; -import {runCodeGraphCompact, runCodeGraphIndex, runCodeGraphPurge, runCodeGraphRepair} from './code_graph/commands.js'; +import { + handleManagerWorksetRequest, + isManagerWorksetApiPath, + managerWorksetRequestAllowedDuringMaintenance, +} from './manager_worksets.js'; +import { + cleanupMode, + consolidationAgent, + memoryKind, + memoryStatus, + optionalNonEmptyQuery, + optionalNonNegativeIntegerQuery, + optionalPositiveIntegerQuery, + optionalString, + requireConfirm, + requiredQuery, + requireString, + requireStringArray, +} from './manager_request_inputs.js'; +import {runCodeGraphIndex, runCodeGraphPurge, runCodeGraphRepair} from './code_graph/commands.js'; +import { + compactCodeGraphStorageIsolated, + runCodeGraphAutomaticCompactionLoop, + type CodeGraphAutomaticCompactionStatus, +} from './code_graph/automatic_compaction.js'; import {inspectAllCodeGraphsLocal} from './code_graph/diagnostics.js'; import {readAllCodeGraphBuildStatuses} from './code_graph/build_status.js'; import { @@ -113,6 +137,12 @@ interface ManagerDirectoryEntry { readonly isFile: () => boolean; } +class ManagerOperationError extends Error { + readonly _tag = 'ManagerOperationError' as const; +} +function managerOperationError(cause: unknown): ManagerOperationError { + return cause instanceof ManagerOperationError ? cause : new ManagerOperationError(errorMessage(cause), {cause}); +} const pathJoin = Effect.fn('manager.pathJoin')(function* (...parts: readonly string[]) { const path = yield* Path.Path; return path.join(...parts); @@ -195,8 +225,10 @@ interface ReadTreeOptions { } interface ApiContext { + readonly automaticCompactionStatus?: Ref.Ref; readonly config: RuntimeConfig; readonly jobs: Map; + readonly worksetScope: Scope.Scope; readonly runEffect?: ManagerEffectPromise; readonly token: string; } @@ -271,33 +303,46 @@ const STATIC_FILES: Readonly< export function runManage(config: RuntimeConfig, options: ManageOptions) { return Effect.scoped( - Effect.gen(function* () { - if (yield* codeGraphMaintenanceIntentActive(config.agentContextHome)) { - return yield* Effect.fail(new Error(GRAPH_MAINTENANCE_BUSY_MESSAGE)); - } - const crypto = yield* Crypto.Crypto; - const lifecycleMaintenance = yield* CodeGraphMaintenanceCoordinator; - const lifecycleTargets = yield* observeCodeGraphLifecycleOpportunityTargets(config.agentContextHome); - yield* runCodeGraphLifecycleOpportunity({ - maintenance: lifecycleMaintenance, - opportunity: 'startup', - targets: lifecycleTargets, - threadnoteHome: config.agentContextHome, - }).pipe(Effect.catch(() => Effect.void)); - const token = Encoding.encodeBase64Url(yield* crypto.randomBytes(24)); - const server = yield* HttpServer.HttpServer; - yield* Effect.addFinalizer(() => releaseManagerGraphSnapshotLeases()); - yield* server.serve(createManagerServer({config, jobs: new Map(), token})); - const actualPort = server.address._tag === 'TcpAddress' ? server.address.port : (options.uiPort ?? 0); - const url = `http://127.0.0.1:${actualPort}/?token=${encodeURIComponent(token)}`; - yield* Console.log(`Threadnote manager: ${url}`); - yield* Console.log('Press Ctrl-C to stop the manager.'); - if (options.open !== false) { - yield* runCommandEffect('open', [url], {allowFailure: true}); - } - return yield* Effect.never; - }), - ).pipe(Effect.provide(BunHttpServer.layer({hostname: '127.0.0.1', port: options.uiPort ?? 0}))); + Layer.build(BunHttpServer.layer({hostname: '127.0.0.1', port: options.uiPort ?? 0})).pipe( + Effect.flatMap(context => + Effect.gen(function* () { + if (yield* codeGraphMaintenanceIntentActive(config.agentContextHome)) { + return yield* Effect.fail(new ManagerOperationError(GRAPH_MAINTENANCE_BUSY_MESSAGE)); + } + const crypto = yield* Crypto.Crypto; + const lifecycleMaintenance = yield* CodeGraphMaintenanceCoordinator; + const lifecycleTargets = yield* observeCodeGraphLifecycleOpportunityTargets(config.agentContextHome); + yield* runCodeGraphLifecycleOpportunity({ + maintenance: lifecycleMaintenance, + opportunity: 'startup', + targets: lifecycleTargets, + threadnoteHome: config.agentContextHome, + }).pipe(Effect.catch(() => Effect.void)); + const token = Encoding.encodeBase64Url(yield* crypto.randomBytes(24)); + const automaticCompactionStatus = yield* Ref.make({state: 'idle'}); + const worksetScope = yield* Scope.Scope; + const server = yield* HttpServer.HttpServer; + yield* Effect.addFinalizer(() => releaseManagerGraphSnapshotLeases()); + yield* server.serve( + createManagerServer({automaticCompactionStatus, config, jobs: new Map(), token, worksetScope}), + ); + yield* Effect.forkScoped( + runCodeGraphAutomaticCompactionLoop(config.agentContextHome, status => + Ref.set(automaticCompactionStatus, status), + ), + ); + const actualPort = server.address._tag === 'TcpAddress' ? server.address.port : (options.uiPort ?? 0); + const url = `http://127.0.0.1:${actualPort}/?token=${encodeURIComponent(token)}`; + yield* Console.log(`Threadnote manager: ${url}`); + yield* Console.log('Press Ctrl-C to stop the manager.'); + if (options.open !== false) { + yield* runCommandEffect('open', [url], {allowFailure: true}); + } + return yield* Effect.never; + }).pipe(Effect.provide(context)), + ), + ), + ); } type ManagerRequestEffect = Effect.Effect; @@ -317,7 +362,7 @@ export function createManagerServer( Effect.flatMap(parsed => typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? Effect.succeed(parsed as Record) - : Effect.fail(new Error('Expected a JSON object body.')), + : Effect.fail(new ManagerOperationError('Expected a JSON object body.')), ), ), headers: request.headers, @@ -361,11 +406,11 @@ export const readManagedMemory = Effect.fn('manager.readManagedMemory')(function assertResourceUri(uri); const path = yield* localPathForMemoryUri(config, uri); if (!path) { - return yield* Effect.fail(new Error(`Manager can only read current-user memory URIs: ${uri}`)); + return yield* Effect.fail(new ManagerOperationError(`Manager can only read current-user memory URIs: ${uri}`)); } const pathStat = yield* lstat(path); if (!pathStat.isFile()) { - return yield* Effect.fail(new Error(`Manager can only read regular memory files: ${uri}`)); + return yield* Effect.fail(new ManagerOperationError(`Manager can only read regular memory files: ${uri}`)); } const content = yield* readFile(path, 'utf8'); const relativePath = (yield* pathRelative(yield* localMemoriesRoot(config), path)) @@ -517,11 +562,28 @@ const handleRequestLegacy = Effect.fn('manager.handleRequestLegacy')(function* ( return; } if ( - isGraphApiPath(url.pathname) && - url.pathname !== '/api/graphs/status' && + ((isGraphApiPath(url.pathname) && url.pathname !== '/api/graphs/status') || + (isManagerWorksetApiPath(url.pathname) && + !managerWorksetRequestAllowedDuringMaintenance(request.method, url.pathname))) && (yield* codeGraphMaintenanceIntentActive(context.config.agentContextHome)) ) { - writeJson(response, 409, {error: GRAPH_MAINTENANCE_BUSY_MESSAGE}); + writeJson(response, 409, { + code: 'maintenance-busy', + error: GRAPH_MAINTENANCE_BUSY_MESSAGE, + retryAfterMilliseconds: 1_000, + }); + return; + } + const worksetResponse = yield* handleManagerWorksetRequest({ + body: request.body, + config: context.config, + contextKey: context, + jobScope: context.worksetScope, + method: request.method, + url, + }); + if (worksetResponse) { + writeJson(response, worksetResponse.status, worksetResponse.body); return; } @@ -535,7 +597,10 @@ const handleRequestLegacy = Effect.fn('manager.handleRequestLegacy')(function* ( return; } if (request.method === 'GET' && url.pathname === '/api/graphs/status') { - writeJson(response, 200, yield* managerGraphBuildCatalog(context.config.agentContextHome)); + const automaticCompaction = context.automaticCompactionStatus + ? yield* Ref.get(context.automaticCompactionStatus) + : undefined; + writeJson(response, 200, yield* managerGraphBuildCatalog(context.config.agentContextHome, automaticCompaction)); return; } if (request.method === 'GET' && url.pathname === '/api/graphs/diagnostics') { @@ -938,7 +1003,7 @@ const readTree: ( const isDir = pathStat.isDirectory(); if (!isDir) { if (!pathStat.isFile()) { - throw new Error(`Manager can only read regular files or directories: ${uri}`); + throw new ManagerOperationError(`Manager can only read regular files or directories: ${uri}`); } const record = options.parseMemoryDocuments === false @@ -1031,7 +1096,7 @@ const writeRawMemory = Effect.fn('manager.writeRawMemory')(function* ( if (isInSharedNamespace(config, uri)) { const teamName = sharedTeamNameForUri(config, uri); if (!teamName) { - throw new Error(`${uri} is not in a configured shared namespace.`); + throw new ManagerOperationError(`${uri} is not in a configured shared namespace.`); } const team = yield* resolveTeam(config, teamName); const existing = yield* readManagedMemory(config, uri); @@ -1070,7 +1135,7 @@ const moveMemory = Effect.fn('manager.moveMemory')(function* ( if (isInSharedNamespace(config, sourceUri)) { const team = sharedTeamNameForUri(config, sourceUri); if (team !== targetTeam) { - throw new Error( + throw new ManagerOperationError( 'Cross-team shared moves are not supported in V1. Copy/unpublish, then publish to the target team.', ); } @@ -1178,7 +1243,7 @@ const removeSharedSource = Effect.fn('manager.removeSharedSource')(function* ( ) { const teamName = sharedTeamNameForUri(config, sourceUri); if (!teamName) { - throw new Error(`${sourceUri} is not a shared memory.`); + throw new ManagerOperationError(`${sourceUri} is not a shared memory.`); } const team = yield* resolveTeam(config, teamName); const ov = NATIVE_RESOURCE_BACKEND; @@ -1197,22 +1262,24 @@ const removeManagedFolder = Effect.fn('manager.removeManagedFolder')(function* ( assertResourceUri(uri); const rootUri = `threadnote://user/${uriSegment(config.user)}/memories`; if (uri === rootUri) { - throw new Error('Refusing to remove the root memories folder.'); + throw new ManagerOperationError('Refusing to remove the root memories folder.'); } if (isInSharedNamespace(config, uri)) { - throw new Error('Shared folders are managed from Sharing. Remove the share or unpublish selected memories.'); + throw new ManagerOperationError( + 'Shared folders are managed from Sharing. Remove the share or unpublish selected memories.', + ); } const path = yield* localPathForMemoryUri(config, uri); if (!path) { - throw new Error(`Manager can only remove current-user memory folders: ${uri}`); + throw new ManagerOperationError(`Manager can only remove current-user memory folders: ${uri}`); } const pathStat = yield* lstat(path); if (!pathStat.isDirectory()) { - throw new Error(`Not a folder: ${uri}`); + throw new ManagerOperationError(`Not a folder: ${uri}`); } const relativePath = yield* pathRelative(yield* localMemoriesRoot(config), path); if (!relativePath || relativePath.startsWith('..') || relativePath.split(yield* pathSeparator).includes('..')) { - throw new Error('Refusing to remove a folder outside the memories tree.'); + throw new ManagerOperationError('Refusing to remove a folder outside the memories tree.'); } const fileUris = yield* fileUrisUnderFolder(config, path); for (const fileUri of fileUris) { @@ -1264,7 +1331,7 @@ const runBulk = Effect.fn('manager.runBulk')(function* ( runEffect, )).output; } else { - return yield* Effect.fail(new Error(`Unsupported bulk action: ${action}`)); + return yield* Effect.fail(new ManagerOperationError(`Unsupported bulk action: ${action}`)); } return output; }), @@ -1287,7 +1354,7 @@ function createConsolidation(context: ApiContext, body: Record) sourceUris: requireStringArray(body.uris, 'uris'), target: targetFromBody(body), }), - catch: cause => (cause instanceof Error ? cause : new Error(String(cause))), + catch: managerOperationError, }); const job: ConsolidationJob = { agent: input.agent, @@ -1323,10 +1390,10 @@ const applyConsolidation = Effect.fn('manager.applyConsolidation')(function* ( ) { const job = jobs.get(id); if (!job) { - throw new Error('Consolidation job not found.'); + throw new ManagerOperationError('Consolidation job not found.'); } if (job.status !== 'completed' || !job.draft) { - throw new Error('Consolidation job is not completed.'); + throw new ManagerOperationError('Consolidation job is not completed.'); } const draft = optionalString(body.draft) ?? job.draft; const target = targetFromBody({...job.target, ...body}); @@ -1374,7 +1441,7 @@ function runConsolidationAgent( return ( native ?? (yield* Effect.fail( - new Error( + new ManagerOperationError( 'No generation model is selected. Install and select one with `threadnote models`, or configure an explicit remote Effect AI provider.', ), )) @@ -1382,12 +1449,14 @@ function runConsolidationAgent( }); } if (agent !== 'codex' && agent !== 'claude') { - return Effect.fail(new Error(`${agent} does not expose a supported non-interactive consolidation mode.`)); + return Effect.fail( + new ManagerOperationError(`${agent} does not expose a supported non-interactive consolidation mode.`), + ); } return Effect.gen(function* () { const executable = yield* findExecutable([agent]); if (!executable) { - return yield* Effect.fail(new Error(`${agent} executable was not found.`)); + return yield* Effect.fail(new ManagerOperationError(`${agent} executable was not found.`)); } return yield* Effect.scoped( Effect.gen(function* () { @@ -1403,12 +1472,14 @@ function runConsolidationAgent( }); if (result.exitCode !== 0) { return yield* Effect.fail( - new Error(result.stderr.trim() || result.stdout.trim() || `${agent} exited with ${result.exitCode}`), + new ManagerOperationError( + result.stderr.trim() || result.stdout.trim() || `${agent} exited with ${result.exitCode}`, + ), ); } const draft = result.stdout.trim(); if (!draft) { - return yield* Effect.fail(new Error(`${agent} returned an empty consolidation draft.`)); + return yield* Effect.fail(new ManagerOperationError(`${agent} returned an empty consolidation draft.`)); } return draft; }), @@ -1423,7 +1494,7 @@ export function consolidationAgentScript(agent: AgentClient, executable: string) if (agent === 'claude') { return `${shellQuote(executable)} --print --permission-mode default < "$1"`; } - throw new Error(`${agent} does not expose a supported non-interactive consolidation mode.`); + throw new ManagerOperationError(`${agent} does not expose a supported non-interactive consolidation mode.`); } function consolidationPrompt(sources: readonly {readonly content: string; readonly node: ManagerTreeNode}[]): string { @@ -1468,7 +1539,7 @@ const shareSummaries = Effect.fn('manager.shareSummaries')(function* (config: Ru const collectManagerDoctorChecks = Effect.fn('manager.collectManagerDoctorChecks')(function* (config: RuntimeConfig) { const threadnote = yield* findExecutable(['threadnote']); if (!threadnote) { - return collectDoctorChecks(config, {}); + return yield* collectDoctorChecks(config, {}); } const result = yield* runCommand( threadnote, @@ -1526,13 +1597,13 @@ const runManagerGraphAction = Effect.fn('manager.runGraphAction')(function* ( ) { const action = yield* Effect.try({ try: () => requireString(body.action, 'action'), - catch: error => error, + catch: managerOperationError, }); const dryRun = body.dryRun === true; if (!dryRun && ['compact', 'purge', 'purge-all', 'purge-obsolete', 'remove-view', 'repair'].includes(action)) { yield* Effect.try({ try: () => requireConfirm(body), - catch: error => error, + catch: managerOperationError, }); } if (action === 'repair') { @@ -1551,7 +1622,7 @@ const runManagerGraphAction = Effect.fn('manager.runGraphAction')(function* ( } const checkoutId = yield* Effect.try({ try: () => requireGraphIdentity(body.checkoutId, 'checkoutId'), - catch: error => error, + catch: managerOperationError, }); if (action === 'purge') { return yield* runCaptured( @@ -1564,18 +1635,20 @@ const runManagerGraphAction = Effect.fn('manager.runGraphAction')(function* ( } const worktreeId = yield* Effect.try({ try: () => requireGraphIdentity(body.worktreeId, 'worktreeId'), - catch: error => error, + catch: managerOperationError, }); if (action === 'remove-view') { const expectedSnapshotId = yield* Effect.try({ try: () => requireGraphSnapshotIdentity(body.expectedSnapshotId), - catch: error => error, + catch: managerOperationError, }); const target = {checkoutId, snapshotId: expectedSnapshotId, worktreeId}; const approvalDigest = yield* managerGraphViewRemovalApprovalDigest(target); if (!dryRun && body.approvalDigest !== approvalDigest) { return yield* Effect.fail( - new Error('Preview this exact graph view removal and provide its approval digest before applying.'), + new ManagerOperationError( + 'Preview this exact graph view removal and provide its approval digest before applying.', + ), ); } const path = yield* Path.Path; @@ -1622,14 +1695,38 @@ const runManagerGraphAction = Effect.fn('manager.runGraphAction')(function* ( } const repositoryId = yield* Effect.try({ try: () => requireGraphIdentity(body.repositoryId, 'repositoryId'), - catch: error => error, + catch: managerOperationError, }); const expectedIdentity = {checkoutId, repositoryId, worktreeId} satisfies RepositoryIdentityExpectation; const cwd = yield* resolveManagerGraphActionCwd(config.agentContextHome, expectedIdentity, optionalString(body.cwd)); switch (action) { case 'compact': return yield* runCaptured( - () => runCodeGraphCompact(config, {cwd, dryRun, expectedIdentity, force: body.force === true}), + () => + compactCodeGraphStorageIsolated(config.agentContextHome, checkoutId, { + force: body.force === true, + operation: dryRun ? 'probe' : 'compact', + }).pipe( + Effect.flatMap(summary => + summary.action === 'compacted' + ? Console.log( + `Compacted the selected graph in an isolated process and reclaimed ${summary.reclaimedBytes.toLocaleString()} bytes.`, + ) + : summary.action === 'would-compact' + ? Console.log( + `The selected graph is eligible for isolated compaction; estimated opportunity ${summary.reclaimedBytes.toLocaleString()} bytes.`, + ) + : summary.action === 'deferred' + ? Console.log( + `Graph compaction was deferred because ${ + summary.reason === 'active-build' + ? 'a graph build is active' + : 'another maintenance operation is active' + }.`, + ) + : Console.log(`Graph compaction completed with result: ${summary.action}.`), + ), + ), runEffect, ); case 'index': @@ -1638,7 +1735,7 @@ const runManagerGraphAction = Effect.fn('manager.runGraphAction')(function* ( runEffect, ); default: - return yield* Effect.fail(new Error(`Unsupported graph Manager action: ${action}`)); + return yield* Effect.fail(new ManagerOperationError(`Unsupported graph Manager action: ${action}`)); } }); @@ -1650,13 +1747,15 @@ const resolveManagerGraphActionCwd = Effect.fn('manager.resolveGraphActionCwd')( if (suppliedCwd) { const path = yield* Path.Path; if (!path.isAbsolute(suppliedCwd)) { - return yield* Effect.fail(new Error('Supply cwd as an absolute local worktree path.')); + return yield* Effect.fail(new ManagerOperationError('Supply cwd as an absolute local worktree path.')); } const {identity} = yield* resolveAndRecordCodeGraphLocalAssociation(threadnoteHome, suppliedCwd, { validateIdentity: identity => repositoryIdentityMatchesExpectation(identity, expectedIdentity) ? Effect.void - : Effect.fail(new Error('The supplied worktree path does not match the selected graph identity.')), + : Effect.fail( + new ManagerOperationError('The supplied worktree path does not match the selected graph identity.'), + ), }); return identity.repoRoot; } @@ -1666,7 +1765,9 @@ const resolveManagerGraphActionCwd = Effect.fn('manager.resolveGraphActionCwd')( validateIdentity: identity => repositoryIdentityMatchesExpectation(identity, expectedIdentity) ? Effect.void - : Effect.fail(new Error('The persisted worktree path no longer matches the selected graph identity.')), + : Effect.fail( + new ManagerOperationError('The persisted worktree path no longer matches the selected graph identity.'), + ), }).pipe(Effect.option); if (Option.isSome(observed)) return observed.value.identity.repoRoot; } @@ -1685,26 +1786,31 @@ const resolveManagerGraphActionCwd = Effect.fn('manager.resolveGraphActionCwd')( validateIdentity: identity => repositoryIdentityMatchesExpectation(identity, expectedIdentity) ? Effect.void - : Effect.fail(new Error('Manager graph context no longer matches the selected graph identity.')), + : Effect.fail( + new ManagerOperationError('Manager graph context no longer matches the selected graph identity.'), + ), }, ).pipe(Effect.option); if (Option.isSome(observed)) return observed.value.identity.repoRoot; } return yield* Effect.fail( - new Error('The selected graph has no current local worktree target. Supply cwd and refresh graph diagnostics.'), + new ManagerOperationError( + 'The selected graph has no current local worktree target. Supply cwd and refresh graph diagnostics.', + ), ); }); function requireGraphIdentity(value: unknown, name: string): string { const identity = requireString(value, name); - if (!/^[0-9a-f]{64}$/.test(identity)) throw new Error(`Provide ${name} as a 64-character graph identity.`); + if (!/^[0-9a-f]{64}$/.test(identity)) + throw new ManagerOperationError(`Provide ${name} as a 64-character graph identity.`); return identity; } function requireGraphSnapshotIdentity(value: unknown): string { const identity = requireString(value, 'expectedSnapshotId'); if (!/^cgsn_[0-9a-f]{40}(?:-direct|-full-[0-9a-f]{16})?$/.test(identity)) { - throw new Error('Provide expectedSnapshotId as an exact code graph snapshot identity.'); + throw new ManagerOperationError('Provide expectedSnapshotId as an exact code graph snapshot identity.'); } return identity; } @@ -1752,7 +1858,7 @@ function sharedMemoryUriFor( }, ): string { if (metadata.kind !== 'durable') { - throw new Error('Only durable memories can be moved into shared team memory.'); + throw new ManagerOperationError('Only durable memories can be moved into shared team memory.'); } return `threadnote://user/${uriSegment(config.user)}/memories/shared/${uriSegment(team)}/durable/projects/${uriSegment(metadata.project)}/${uriSegment(metadata.topic)}.md`; } @@ -1820,7 +1926,7 @@ function isMissingPathError(err: unknown): boolean { const localPathToMemoryUri = Effect.fn('manager.localPathToMemoryUri')(function* (config: RuntimeConfig, path: string) { const relativePath = yield* pathRelative(yield* localMemoriesRoot(config), path); if (!relativePath || relativePath.startsWith('..') || relativePath.split(yield* pathSeparator).includes('..')) { - throw new Error(`Path is outside the memories tree: ${path}`); + throw new ManagerOperationError(`Path is outside the memories tree: ${path}`); } return `threadnote://user/${uriSegment(config.user)}/memories/${relativePath.split(yield* pathSeparator).join('/')}`; }); @@ -1874,59 +1980,6 @@ function writeJson(response: ManagerResponseSink, statusCode: number, body: unkn }); } -function requiredQuery(url: URL, name: string): string { - const value = url.searchParams.get(name); - if (!value) { - throw new Error(`Missing query parameter: ${name}`); - } - return value; -} - -function optionalPositiveIntegerQuery(url: URL, name: string): Option.Option { - return Option.fromNullishOr(url.searchParams.get(name)).pipe( - Option.map(value => Number(value)), - Option.filter(value => Number.isSafeInteger(value) && value > 0), - ); -} - -function optionalNonNegativeIntegerQuery(url: URL, name: string): Option.Option { - return Option.fromNullishOr(url.searchParams.get(name)).pipe( - Option.map(value => Number(value)), - Option.filter(value => Number.isSafeInteger(value) && value >= 0), - ); -} - -function optionalNonEmptyQuery(url: URL, name: string): Option.Option { - return Option.fromNullishOr(url.searchParams.get(name)).pipe( - Option.map(value => value.trim()), - Option.filter(value => value.length > 0), - ); -} - -function requireString(value: unknown, name: string): string { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`Provide ${name}.`); - } - return value; -} - -function optionalString(value: unknown): string | undefined { - return typeof value === 'string' && value.trim().length > 0 ? value : undefined; -} - -function requireStringArray(value: unknown, name: string): readonly string[] { - if (!Array.isArray(value) || value.length === 0 || !value.every(item => typeof item === 'string')) { - throw new Error(`Provide ${name} as a non-empty string array.`); - } - return value; -} - -function requireConfirm(body: Record): void { - if (body.confirm !== true) { - throw new Error('Set confirm=true for this action.'); - } -} - function targetFromBody(body: Record): TargetMemoryInput { return { kind: memoryKind(body.kind), @@ -1938,34 +1991,6 @@ function targetFromBody(body: Record): TargetMemoryInput { }; } -function memoryKind(value: unknown): MemoryKind | undefined { - return value === 'durable' || - value === 'handoff' || - value === 'incident' || - value === 'preference' || - value === 'smoke' - ? value - : undefined; -} - -function memoryStatus(value: unknown): MemoryStatus | undefined { - return value === 'active' || value === 'archived' || value === 'superseded' ? value : undefined; -} - function isRawMemoryDocument(text: string): boolean { return text.startsWith('MEMORY\n') || text.startsWith('HANDOFF\n'); } - -function consolidationAgent(value: string): ConsolidationAgent { - if (value === 'codex' || value === 'claude' || value === 'cursor' || value === 'copilot' || value === 'effect-ai') { - return value; - } - throw new Error(`Unsupported consolidation agent: ${value}`); -} - -function cleanupMode(value: unknown): 'archive' | 'forget' | 'keep' { - if (value === 'forget' || value === 'keep') { - return value; - } - return 'archive'; -} diff --git a/src/manager_graph.tsx b/src/manager_graph.tsx index d9d5ac66..a9b03c18 100644 --- a/src/manager_graph.tsx +++ b/src/manager_graph.tsx @@ -1,4939 +1,73 @@ -import React, {useEffect, useMemo, useRef, useState} from 'react'; -import * as THREE from 'three'; -import type {CodeGraphLocalDiagnosticsReport} from './code_graph/diagnostics.js'; -import type {CodeGraphLocalAssociation} from './code_graph/local_provenance.js'; -import type {CodeGraphMaintenanceStatus} from './code_graph/maintenance_gate.js'; -import {compareCodeUnits} from './code_graph/ordering.js'; -import { - CODE_GRAPH_SLOW_FILE_THRESHOLD_MILLISECONDS, - CODE_GRAPH_TOP_SLOW_FILE_LIMIT, -} from './code_graph/progress_telemetry.js'; -import { - MANAGER_GRAPH_DEFAULT_EDGE_LIMIT, - MANAGER_GRAPH_DEFAULT_NODE_LIMIT, - MANAGER_GRAPH_MAX_EDGE_LIMIT, - MANAGER_GRAPH_MAX_NODE_LIMIT, - type ManagerGraphVisualizationLimits, -} from './manager_graph_limits.js'; -import {type ManagerDialogOptions, useOptionalManagerDialogs} from './manager_dialog.js'; - -interface GraphProject { - readonly buildSystem?: string; - readonly fileCount?: number; - readonly id: string; - readonly kind?: string; - readonly label: string; - readonly model?: 'component' | 'facet' | 'legacy-fallback'; - readonly provenance?: string; - readonly symbolCount?: number; - readonly workspaceId?: string; -} - -interface GraphWorkspaceDescriptor { - readonly buildSystem: string; - readonly id: string; - readonly name: string; - readonly root: string; -} - -interface GraphSnapshot { - readonly commit: string; - readonly completedAt?: string; - readonly dirty: boolean; - readonly edgeCount: number; - readonly fileCount: number; - readonly id: string; - readonly symbolCount: number; -} - -export interface GraphRepository { - readonly accounting: { - readonly attributedSymbols: number; - readonly componentSymbols: number; - readonly fallbackSymbols: number; - readonly omittedSymbols: number; - readonly totalSymbols: number; - }; - readonly activatedAt?: string; - readonly checkoutId: string; - readonly displayName: string; - readonly id: string; - readonly label: string; - readonly localAssociation: CodeGraphLocalAssociation; - readonly metrics: 'complete' | 'deferred'; - readonly model: 'legacy-fallback' | 'workspace'; - readonly projectCount: number; - readonly projects: readonly GraphProject[]; - readonly projectsTruncated: boolean; - readonly snapshot: GraphSnapshot; - readonly worktreeId: string; - readonly workspaceCount: number; - readonly workspaces: readonly GraphWorkspaceDescriptor[]; - readonly workspacesTruncated: boolean; -} - -export interface GraphRepositoryGroup { - readonly defaultViewId: string; - readonly displayName: string; - readonly id: string; - readonly repositoryId: string; - readonly views: readonly GraphRepository[]; - readonly viewsTruncated: boolean; -} - -export interface GraphCatalogDiagnostic { - readonly checkoutId: string; - readonly code: 'lease-deferred' | 'lease-failed' | 'no-ready-snapshot' | 'unreadable-database'; - readonly message: string; -} - -export interface GraphCatalog { - readonly builds: readonly GraphBuildStatus[]; - readonly catalogRevision?: string; - readonly diagnostics: readonly GraphCatalogDiagnostic[]; - readonly lifecyclePending?: boolean; - readonly maintenance?: CodeGraphMaintenanceStatus; - readonly repositories: readonly GraphRepositoryGroup[]; - readonly waiterCount: number; - readonly waiters: readonly GraphBuildStatus[]; -} - -export type GraphAdministrationAction = - | { - readonly action: 'compact' | 'index'; - readonly checkoutId: string; - readonly cwd?: string; - readonly dryRun?: boolean; - readonly force?: boolean; - readonly full?: boolean; - readonly repositoryId: string; - readonly worktreeId: string; - } - | { - readonly action: 'purge' | 'purge-obsolete'; - readonly checkoutId: string; - readonly dryRun?: boolean; - } - | { - readonly action: 'remove-view'; - readonly checkoutId: string; - readonly dryRun?: boolean; - readonly expectedSnapshotId: string; - readonly worktreeId: string; - } - | {readonly action: 'purge-all'; readonly dryRun?: boolean} - | {readonly action: 'repair'; readonly deep?: boolean; readonly dryRun?: boolean}; - -type GraphWorktreeAdministrationAction = Extract; - -export function graphAdministrationTarget( - checkoutId: string, - view: {readonly repository: {readonly repositoryId: string}; readonly worktreeId: string}, -): Pick { - return {checkoutId, repositoryId: view.repository.repositoryId, worktreeId: view.worktreeId}; -} - -export function graphViewRemovalTarget( - checkoutId: string, - view: {readonly snapshot: {readonly id: string}; readonly worktreeId: string}, -): Pick< - Extract, - 'checkoutId' | 'expectedSnapshotId' | 'worktreeId' -> { - return {checkoutId, expectedSnapshotId: view.snapshot.id, worktreeId: view.worktreeId}; -} - -export interface GraphCatalogPage { - readonly projectOffset: number; - readonly query: string; - readonly repository: GraphRepository; - readonly workspaceOffset: number; -} - -export interface GraphViewPage { - readonly hasMore: boolean; - readonly offset: number; - readonly query: string; - readonly repositories: readonly GraphRepositoryGroup[]; -} - -export interface GraphBuildStatus { - readonly activation?: { - readonly activity: { - readonly elapsedMilliseconds: number; - readonly rows?: number; - readonly stage: GraphActivationStage; - readonly stageElapsedMilliseconds: number; - readonly startedAt: string; - readonly state: 'completed' | 'progress' | 'started'; - readonly transactionMilliseconds?: number; - }; - }; - readonly activity?: { - readonly batchCompleted: number; - readonly batchTotal: number; - readonly bytes: number; - readonly classifier?: string; - readonly degraded?: boolean; - readonly factsBytes?: number; - readonly language: string; - readonly parseMilliseconds?: number; - readonly persistMilliseconds?: number; - readonly relations?: number; - readonly role?: string; - readonly sizeBucket?: '0-16KiB' | '16-64KiB' | '64-256KiB' | '256KiB-1MiB' | '>1MiB'; - readonly stage: 'extracting' | 'persisting' | 'reading'; - readonly symbols?: number; - }; - readonly buildId: string; - readonly coordination?: { - readonly lockVerified: boolean; - readonly progressSilent?: boolean; - readonly role: 'history' | 'owner' | 'waiter'; - }; - readonly counters: { - readonly accepted?: number; - readonly completed?: number; - readonly edges?: number; - readonly excluded?: number; - readonly pagesCompleted?: number; - readonly reused?: number; - readonly resolved?: number; - readonly rowsDeleted?: number; - readonly skipped?: number; - readonly symbols?: number; - readonly total?: number; - readonly unit?: string; - }; - readonly error?: {readonly summary: string}; - readonly eta?: { - readonly basis?: 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes'; - readonly confidence: 'high' | 'low' | 'medium'; - readonly remainingMilliseconds: number; - }; - readonly extraction?: { - readonly completedFiles: number; - readonly metrics?: { - readonly factsBytesCompleted: number; - readonly sourceBytesCompleted: number; - readonly sourceBytesTotal: number; - readonly workUnitsCompleted: number; - readonly workUnitsTotal: number; - }; - readonly slowFiles: number; - readonly topSlowFiles: readonly { - readonly classifier: string; - readonly degraded?: boolean; - readonly durationMilliseconds: number; - readonly extension: string; - readonly factsBytes?: number; - readonly language: string; - readonly pathHash: string; - readonly relations?: number; - readonly role: string; - readonly sizeBucket: '0-16KiB' | '16-64KiB' | '64-256KiB' | '256KiB-1MiB' | '>1MiB'; - readonly sourceBytes: number; - readonly symbols?: number; - }[]; - }; - readonly identity: { - readonly checkoutId: string; - readonly commit: string; - readonly displayName?: string; - readonly repositoryId: string; - readonly worktreeId: string; - }; - readonly managerContext?: { - readonly worktreePath: string; - }; - readonly observation: { - readonly heartbeatAgeMilliseconds: number; - readonly liveness: 'abandoned' | 'active' | 'completed' | 'failed' | 'stalled'; - }; - readonly materialization?: { - readonly activity?: { - readonly batchCompleted: number; - readonly batchTotal: number; - readonly cachedFactBytes?: number; - readonly elapsedMilliseconds?: number; - readonly factsBytes?: number; - readonly rows?: GraphMaterializationRows; - readonly sourceBytes: number; - readonly stage: GraphMaterializationStage; - readonly stageElapsedMilliseconds?: number; - readonly startedAt: string; - readonly transactionMilliseconds?: number; - }; - readonly metrics?: { - readonly attributionMilliseconds?: number; - readonly batchesCompleted: number; - readonly batchesTotal: number; - readonly cachedFactBytesCompleted?: number; - readonly cachedFactBytesTotal?: number; - readonly fallbackReason?: string; - readonly factsBytesCompleted?: number; - readonly factsBytesTotal?: number; - readonly loadingMilliseconds?: number; - readonly mode?: 'full' | 'incremental-clean' | 'incremental-overlay'; - readonly rows?: GraphMaterializationRows; - readonly sourceBytesCompleted: number; - readonly sourceBytesTotal: number; - readonly stageMilliseconds?: Readonly>>; - readonly storage?: GraphMaterializationStorage; - readonly transactionMilliseconds?: number; - }; - }; - readonly owner: {readonly processId: number; readonly processStartIdentity?: string}; - readonly phase: string; - readonly request?: {readonly key: string}; - readonly resolution?: { - readonly activity: { - readonly aliasesDiscovered: number; - readonly elapsedMilliseconds: number; - readonly matchingMilliseconds: number; - readonly pageCompleted: number; - readonly pageTotal: number; - readonly pagesCompleted: number; - readonly pass: number; - readonly referencesCompleted: number; - readonly referencesExamined: number; - readonly referencesTotal: number; - readonly resolved: number; - readonly startedAt: string; - readonly transactionMilliseconds: number; - }; - }; - readonly result?: {readonly snapshotId: string}; - readonly state: 'completed' | 'failed' | 'queued' | 'running'; - readonly subphase?: string; - readonly timings?: { - readonly extractionMilliseconds: number; - readonly persistenceMilliseconds: number; - readonly readingMilliseconds: number; - }; - readonly timestamps: { - readonly heartbeatAt: string; - readonly lastProgressAt: string; - readonly startedAt: string; - }; -} - -type GraphActivationStage = - | 'checkpointing-snapshot' - | 'committing-snapshot' - | 'copying-edges' - | 'copying-files' - | 'copying-lookup-keys' - | 'copying-reexports' - | 'copying-symbols' - | 'copying-terms' - | 'copying-workspace' - | 'recording-completion' - | 'validating-input'; - -type GraphMaterializationStage = - | 'attributing' - | 'committing' - | 'loading-cache' - | 'preparing-rows' - | 'writing-analysis' - | 'writing-candidates' - | 'writing-edges' - | 'writing-facts' - | 'writing-lookups' - | 'writing-references' - | 'writing-receipt' - | 'writing-symbols' - | 'writing-terms'; - -interface GraphMaterializationRows { - readonly deduplicatedEdges?: number; - readonly deduplicatedReferences?: number; - readonly edges?: number; - readonly lookupKeys?: number; - readonly referenceCandidates?: number; - readonly references?: number; - readonly reexports?: number; - readonly symbols?: number; - readonly terms?: number; -} - -interface GraphMaterializationStorage { - readonly availableBytes?: number; - readonly durableAvailableBytes?: number; - readonly durableDatabaseBytes?: number; - readonly durableDatabaseFileBytes?: number; - readonly durableDatabaseFileHighWaterBytes?: number; - readonly durableDatabaseGrowthBytes?: number; - readonly durableDatabaseGrowthHighWaterBytes?: number; - readonly durableDatabaseHighWaterBytes?: number; - readonly durableDatabaseStartBytes?: number; - readonly durableFilesystemBytes?: number; - readonly durableFilesystemHighWaterBytes?: number; - readonly durableJournalBytes?: number; - readonly durableJournalHighWaterBytes?: number; - readonly durableSharedMemoryBytes?: number; - readonly durableSharedMemoryHighWaterBytes?: number; - readonly durableWalBytes?: number; - readonly durableWalHighWaterBytes?: number; - readonly estimateBasis?: 'cached-fact-bytes' | 'final-fact-bytes' | 'source-bytes-fallback'; - readonly estimatedConcurrentBuildBytes?: number; - readonly estimatedDurableFilesystemRequiredBytes?: number; - readonly estimatedDurableSnapshotBytes?: number; - readonly estimatedJournalBytes?: number; - readonly estimatedRequiredBytes?: number; - readonly estimatedTemporaryFilesystemRequiredBytes?: number; - readonly estimatedTemporaryDatabaseBytes?: number; - readonly filesystemsShared?: boolean; - readonly materializationMode?: 'direct-persistent' | 'temporary-staged'; - readonly temporaryAvailableBytes?: number; - readonly temporaryDatabaseBytes: number; - readonly temporaryDatabaseHighWaterBytes: number; -} - -export function graphBuildIsActive(build: GraphBuildStatus): boolean { - return ( - (build.state === 'queued' || build.state === 'running') && - build.observation.liveness === 'active' && - build.coordination?.role !== 'history' - ); -} - -export function graphBuildShouldDisplay(build: GraphBuildStatus): boolean { - return build.state === 'failed' || graphBuildIsActive(build); -} - -const GRAPH_ADMINISTRATION_JOB_LIMIT = 4; - -export interface GraphAdministrationJobSelection { - readonly hiddenCount: number; - readonly jobs: readonly GraphBuildStatus[]; - readonly total: number; -} - -/** Keep administration cards focused on bounded, actionable build state. */ -export function graphAdministrationJobSelection( - builds: readonly GraphBuildStatus[], - waiters: readonly GraphBuildStatus[], -): GraphAdministrationJobSelection { - const unique = new Map(); - for (const job of [...builds, ...waiters]) { - if (graphBuildShouldDisplay(job) && !unique.has(job.buildId)) unique.set(job.buildId, job); - } - const relevant = [...unique.values()].sort(compareGraphAdministrationJob); - const jobs = relevant.slice(0, GRAPH_ADMINISTRATION_JOB_LIMIT); - return {hiddenCount: relevant.length - jobs.length, jobs, total: relevant.length}; -} - -function compareGraphAdministrationJob(left: GraphBuildStatus, right: GraphBuildStatus): number { - const priority = (job: GraphBuildStatus) => (job.state === 'running' ? 0 : job.state === 'queued' ? 1 : 2); - return ( - priority(left) - priority(right) || - (Date.parse(right.timestamps.lastProgressAt) || 0) - (Date.parse(left.timestamps.lastProgressAt) || 0) || - compareCodeUnits(left.buildId, right.buildId) - ); -} - -function graphAdministrationInventorySummary( - summary: Pick, -): string { - return [ - graphAdministrationCount(summary.databaseCount, 'graph database'), - graphAdministrationCount(summary.readySnapshotCount, 'stored ready snapshot'), - graphAdministrationCount(summary.viewCount, 'active worktree view'), - ].join(' · '); -} - -function graphAdministrationCount(count: number, singular: string): string { - return `${count.toLocaleString()} ${singular}${count === 1 ? '' : 's'}`; -} - -export interface GraphBuildTarget { - readonly repositoryLabel: string; - readonly worktreeLabel: string; -} - -export interface GraphBuildConcurrencyState { - readonly activeTargetCommit?: string; - readonly latestTargetCommit: string; - readonly queuedRequests: number; - readonly readySnapshotCommit?: string; - readonly staleReady: boolean; -} - -export function graphBuildTarget( - build: GraphBuildStatus, - repositories: readonly GraphRepositoryGroup[], -): GraphBuildTarget { - const repository = repositories.find(candidate => candidate.repositoryId === build.identity.repositoryId); - const view = repository?.views.find( - candidate => - candidate.checkoutId === build.identity.checkoutId && candidate.worktreeId === build.identity.worktreeId, - ); - const fallbackName = build.identity.displayName?.trim(); - const repositoryLabel = repository - ? graphRepositoryOptionLabel(repository, repositories) - : fallbackName - ? `${fallbackName} · repository ${shortGraphIdentity(build.identity.repositoryId)}` - : `Repository ${shortGraphIdentity(build.identity.repositoryId)}`; - return { - repositoryLabel, - worktreeLabel: - view?.localAssociation.displayPath ?? - view?.label ?? - `Checkout ${shortGraphIdentity(build.identity.checkoutId)} · worktree ${shortGraphIdentity( - build.identity.worktreeId, - )}`, - }; -} - -/** - * Summarize only observed concurrency facts. File locks do not promise FIFO, so - * waiters are counted without claiming an execution position. The most recent - * request is the latest requested target, independent of input ordering. - */ -export function graphBuildConcurrencyState( - build: GraphBuildStatus, - waiters: readonly GraphBuildStatus[], - repositories: readonly GraphRepositoryGroup[], -): GraphBuildConcurrencyState { - const matchingWaiters = waiters.filter( - waiter => - waiter.buildId !== build.buildId && - waiter.identity.checkoutId === build.identity.checkoutId && - waiter.identity.worktreeId === build.identity.worktreeId, - ); - const latest = [build, ...matchingWaiters].sort(compareGraphBuildRequest)[matchingWaiters.length]!; - const repository = repositories.find(candidate => candidate.repositoryId === build.identity.repositoryId); - const ready = repository?.views.find( - candidate => - candidate.checkoutId === build.identity.checkoutId && candidate.worktreeId === build.identity.worktreeId, - ); - const queuedRequests = matchingWaiters.length + (build.state === 'queued' ? 1 : 0); - const readySnapshotCommit = ready?.snapshot.commit; - return { - ...(build.state === 'running' ? {activeTargetCommit: build.identity.commit} : {}), - latestTargetCommit: latest.identity.commit, - queuedRequests, - ...(readySnapshotCommit === undefined ? {} : {readySnapshotCommit}), - staleReady: readySnapshotCommit !== undefined && !graphCommitMatches(readySnapshotCommit, latest.identity.commit), - }; -} - -function compareGraphBuildRequest(left: GraphBuildStatus, right: GraphBuildStatus): number { - const leftStartedAt = Date.parse(left.timestamps.startedAt) || 0; - const rightStartedAt = Date.parse(right.timestamps.startedAt) || 0; - return leftStartedAt - rightStartedAt || compareCodeUnits(left.buildId, right.buildId); -} - -function graphCommitMatches(left: string, right: string): boolean { - return left === right || left.startsWith(right) || right.startsWith(left); -} - -export function graphStatusPollDelay( - builds: readonly GraphBuildStatus[], - maintenance?: CodeGraphMaintenanceStatus, - lifecyclePending = false, -): number { - return builds.some(graphBuildIsActive) || maintenance !== undefined || lifecyclePending ? 1_000 : 5_000; -} - -export function graphMaintenanceStatusLabel(status: CodeGraphMaintenanceStatus): string { - const operation = status.operation === 'selected-snapshot-purge' ? 'Selected snapshot purge' : 'Graph maintenance'; - const phases: Record = { - 'acquiring-gates': 'acquiring safety gates', - 'retiring-and-cleaning': 'retiring snapshot and advancing cleanup', - 'status-unavailable': 'working; detailed status unavailable', - 'verifying-graph': 'rechecking graph safety evidence', - 'verifying-vectors': 'rechecking vector safety evidence', - 'waiting-builders': 'waiting for graph builders', - working: 'working', - }; - return `${operation} · ${phases[status.phase]}`; -} - -export function graphCompletedBuildResultIdentity(build: GraphBuildStatus): string | undefined { - return build.state === 'completed' && build.result !== undefined - ? `${build.buildId}:${build.result.snapshotId}` - : undefined; -} - -export function graphStatusRequiresCatalogRefresh( - catalog: GraphCatalog | undefined, - builds: readonly GraphBuildStatus[], - acknowledgedResults: ReadonlySet = new Set(), - observedCatalogRevision?: string, -): boolean { - if ( - catalog !== undefined && - observedCatalogRevision !== undefined && - catalog.catalogRevision !== observedCatalogRevision - ) { - return true; - } - if (!catalog) { - return builds.some(build => { - const identity = graphCompletedBuildResultIdentity(build); - return identity !== undefined && !acknowledgedResults.has(identity); - }); - } - return builds.some(build => { - const identity = graphCompletedBuildResultIdentity(build); - const resultVisible = catalog.repositories.some( - repository => - repository.repositoryId === build.identity.repositoryId && - repository.views.some( - view => - view.checkoutId === build.identity.checkoutId && - view.worktreeId === build.identity.worktreeId && - view.snapshot.id === build.result?.snapshotId, - ), - ); - return identity !== undefined && !acknowledgedResults.has(identity) && !resultVisible; - }); -} - -export function graphDiagnosticsRequiresCatalogRefresh( - diagnosticsCatalogRevision: string | undefined, - observedCatalogRevision: string | undefined, - maintenance?: CodeGraphMaintenanceStatus, -): boolean { - return ( - maintenance === undefined && - observedCatalogRevision !== undefined && - diagnosticsCatalogRevision !== observedCatalogRevision - ); -} - -export function graphWaiterCountForBuild(build: GraphBuildStatus, waiters: readonly GraphBuildStatus[]): number { - return waiters.filter( - waiter => - waiter.identity.checkoutId === build.identity.checkoutId && - waiter.identity.worktreeId === build.identity.worktreeId && - waiter.request?.key === build.request?.key, - ).length; -} - -export function resolveGraphSelection( - repositories: readonly GraphRepositoryGroup[], - currentRepositoryId: string, - currentViewId: string, -): {readonly repositoryId: string; readonly viewId: string} { - const repository = repositories.find(candidate => candidate.id === currentRepositoryId) ?? repositories[0]; - if (!repository) return {repositoryId: '', viewId: ''}; - const view = repository.views.find(candidate => candidate.id === currentViewId); - return { - repositoryId: repository.id, - viewId: view?.id ?? repository.defaultViewId ?? repository.views[0]?.id ?? '', - }; -} - -export function graphRepositoryOptionLabel( - repository: GraphRepositoryGroup, - repositories: readonly GraphRepositoryGroup[], -): string { - const collides = repositories.some( - candidate => candidate.id !== repository.id && candidate.displayName === repository.displayName, - ); - return collides ? `${repository.displayName} · ${repository.id.slice(0, 8)}` : repository.displayName; -} - -function shortGraphIdentity(value: string): string { - return value.slice(-8) || 'unknown'; -} - -export function mergeGraphRepositoryGroups( - current: readonly GraphRepositoryGroup[], - additions: readonly GraphRepositoryGroup[], -): readonly GraphRepositoryGroup[] { - const groups = new Map(current.map(group => [group.id, {...group, views: [...group.views]}])); - for (const addition of additions) { - const existing = groups.get(addition.id); - if (!existing) { - groups.set(addition.id, {...addition, views: [...addition.views]}); - continue; - } - const views = new Map(existing.views.map(view => [view.id, view])); - for (const view of addition.views) { - const currentView = views.get(view.id); - views.set(view.id, currentView ? mergeGraphRepository(currentView, view) : view); - } - groups.set(addition.id, { - ...existing, - defaultViewId: existing.defaultViewId || addition.defaultViewId, - views: [...views.values()], - viewsTruncated: existing.viewsTruncated || addition.viewsTruncated, - }); - } - return [...groups.values()].sort( - (left, right) => compareCodeUnits(left.displayName, right.displayName) || compareCodeUnits(left.id, right.id), - ); -} - -export function graphCatalogPageOffsets(input: { - readonly baseRepository?: GraphRepository; - readonly baseRepositoryGroup?: GraphRepositoryGroup; - readonly checkoutId: string; - readonly continuation?: { - readonly projectOffset: number; - readonly viewId: string; - readonly viewOffset: number; - readonly workspaceOffset: number; - }; - readonly viewId: string; -}): {readonly projectOffset: number; readonly viewOffset: number; readonly workspaceOffset: number} { - const continuation = input.continuation?.viewId === input.viewId ? input.continuation : undefined; - return { - projectOffset: - continuation?.projectOffset ?? - input.baseRepository?.projects.filter(project => project.id.startsWith('cgp_')).length ?? - 0, - viewOffset: - continuation?.viewOffset ?? - input.baseRepositoryGroup?.views.filter(view => view.checkoutId === input.checkoutId).length ?? - 0, - workspaceOffset: continuation?.workspaceOffset ?? input.baseRepository?.workspaces.length ?? 0, - }; -} - -function mergeGraphRepository(current: GraphRepository, addition: GraphRepository): GraphRepository { - if (current.snapshot.id !== addition.snapshot.id) { - const currentTime = Date.parse(current.activatedAt ?? current.snapshot.completedAt ?? '') || 0; - const additionTime = Date.parse(addition.activatedAt ?? addition.snapshot.completedAt ?? '') || 0; - return additionTime > currentTime ? addition : current; - } - const projects = new Map(current.projects.map(project => [project.id, project])); - for (const project of addition.projects) projects.set(project.id, project); - const workspaces = new Map(current.workspaces.map(workspace => [workspace.id, workspace])); - for (const workspace of addition.workspaces) workspaces.set(workspace.id, workspace); - return { - ...current, - ...addition, - projectCount: Math.max(current.projectCount, addition.projectCount), - projects: [...projects.values()], - projectsTruncated: current.projectsTruncated || addition.projectsTruncated, - workspaceCount: Math.max(current.workspaceCount, addition.workspaceCount), - workspaces: [...workspaces.values()], - workspacesTruncated: current.workspacesTruncated || addition.workspacesTruncated, - }; -} - -interface GraphNode { - readonly degree: number; - readonly exported?: boolean; - readonly fileCount?: number; - readonly id: string; - readonly kind: string; - readonly label: string; - readonly language?: string; - readonly packageName?: string; - readonly path?: string; - readonly projectId: string; - readonly qualifiedName?: string; - readonly signature?: string; - readonly symbolCount?: number; - readonly type: 'project' | 'symbol'; -} - -export interface GraphEdge { - readonly confidence: number; - readonly count: number; - readonly id: string; - readonly provenance: string; - readonly relation: string; - readonly sourceId: string; - readonly targetId: string; -} - -interface GraphSpan { - readonly column: number; - readonly endColumn: number; - readonly endLine: number; - readonly line: number; -} - -export interface GraphNodeDetail { - readonly node: { - readonly documentation?: string; - readonly exported: boolean; - readonly id: string; - readonly kind: string; - readonly label: string; - readonly language: string; - readonly packageName?: string; - readonly path: string; - readonly projectId: string; - readonly qualifiedName: string; - readonly signature?: string; - readonly span: GraphSpan; - }; - readonly relationships: readonly { - readonly confidence: number; - readonly direction: 'incoming' | 'outgoing'; - readonly evidencePath: string; - readonly evidenceSpan: GraphSpan; - readonly id: string; - readonly provenance: string; - readonly related: { - readonly id?: string; - readonly kind?: string; - readonly label: string; - readonly path?: string; - readonly projectId?: string; - readonly qualifiedName?: string; - }; - readonly relation: string; - }[]; - readonly snapshotId: string; - readonly stats: { - readonly incoming: number; - readonly outgoing: number; - readonly sampledEdges?: number; - readonly summaryTruncated?: boolean; - readonly provenances: readonly {readonly count: number; readonly provenance: string}[]; - readonly relations: readonly { - readonly count: number; - readonly incoming: number; - readonly outgoing: number; - readonly relation: string; - }[]; - readonly truncated: boolean; - }; -} - -export function graphRelationshipCountLabel(count: number, sampled: boolean): string { - return `${sampled ? '≥' : ''}${Math.max(0, count).toLocaleString()}`; -} - -export function graphRelationshipSampleLabel(detail: GraphNodeDetail): string | undefined { - if (detail.stats.summaryTruncated !== true) return undefined; - const sampledEdges = detail.stats.sampledEdges ?? detail.stats.incoming + detail.stats.outgoing; - return `Counts are lower bounds from a ${sampledEdges.toLocaleString()}-edge sample.`; -} - -export function graphDisplayEdges( - edges: readonly GraphEdge[], - selectedNodeId: string | undefined, - focusMode: GraphFocusMode, - relationFilter: string, -): readonly GraphEdge[] { - const related = relationFilter === 'all' ? edges : edges.filter(edge => edge.relation === relationFilter); - if (!selectedNodeId || focusMode === 'all') return related; - return related.filter(edge => { - if (focusMode === 'incoming') return edge.targetId === selectedNodeId; - if (focusMode === 'outgoing') return edge.sourceId === selectedNodeId; - return edge.sourceId === selectedNodeId || edge.targetId === selectedNodeId; - }); -} - -export function graphAnalysisRequestIsCurrent( - currentSequence: number, - requestedSequence: number, - currentScope: string, - requestedScope: string, -): boolean { - return currentSequence === requestedSequence && currentScope === requestedScope; -} - -export function graphRequestIsCurrent( - currentSequence: number, - requestedSequence: number, - currentScope: string, - requestedScope: string, -): boolean { - return currentSequence === requestedSequence && currentScope === requestedScope; -} - -export function graphQueryRequestIsCurrent( - aborted: boolean, - currentSequence: number, - requestedSequence: number, - currentScope: string, - requestedScope: string, - graph: GraphQueryVisualization, - expectedSnapshotId: string, - expectedQuery: string, -): boolean { - return ( - !aborted && - graphRequestIsCurrent(currentSequence, requestedSequence, currentScope, requestedScope) && - graph.repository.snapshot.id === expectedSnapshotId && - graph.query.state === 'ready' && - graph.query.text.trim() === expectedQuery - ); -} - -export interface GraphQueryRequestInput { - readonly expectedQuery: string; - readonly expectedSnapshotId: string; - readonly scope: string; -} - -export type GraphQueryRequestOutcome = - | {readonly graph: GraphQueryVisualization; readonly state: 'accepted'} - | {readonly cause: unknown; readonly state: 'failed'} - | {readonly state: 'cancelled'} - | {readonly graph: GraphQueryVisualization; readonly state: 'stale'}; - -export interface GraphQueryRequestHandle { - readonly cancel: () => void; - readonly isCurrent: () => boolean; - readonly result: Promise; -} - -export interface GraphQueryRequestGate { - readonly cancelCurrent: () => void; - readonly request: ( - input: GraphQueryRequestInput, - load: (signal: AbortSignal) => Promise, - ) => GraphQueryRequestHandle; -} - -/** - * Owns the same supersession boundary used by the Manager graph-query UI. - * - * A new request aborts the previous signal. The sequence, scope, snapshot, and - * query checks remain mandatory even when a loader ignores cancellation and - * eventually delivers a late response. - */ -export function createGraphQueryRequestGate(): GraphQueryRequestGate { - let currentController: AbortController | undefined; - let currentScope = ''; - let currentSequence = 0; - - const cancelCurrent = (): void => { - currentController?.abort(); - currentController = undefined; - currentScope = ''; - currentSequence += 1; - }; - - return { - cancelCurrent, - request: (input, load) => { - currentController?.abort(); - const controller = new AbortController(); - const requestedSequence = currentSequence + 1; - currentController = controller; - currentScope = input.scope; - currentSequence = requestedSequence; - const isCurrent = (): boolean => - currentController === controller && - graphRequestIsCurrent(currentSequence, requestedSequence, currentScope, input.scope); - const cancel = (): void => { - if (!isCurrent()) return; - cancelCurrent(); - }; - let pending: Promise; - try { - pending = load(controller.signal); - } catch (cause) { - pending = Promise.reject(cause); - } - const result = pending.then( - graph => - graphQueryRequestIsCurrent( - controller.signal.aborted, - currentSequence, - requestedSequence, - currentScope, - input.scope, - graph, - input.expectedSnapshotId, - input.expectedQuery, - ) - ? {graph, state: 'accepted'} - : {graph, state: 'stale'}, - cause => - controller.signal.aborted || isAbortError(cause) - ? {state: 'cancelled'} - : isCurrent() - ? {cause, state: 'failed'} - : {state: 'cancelled'}, - ); - return {cancel, isCurrent, result}; - }, - }; -} - -export function graphNodeDetailRequestIsCurrent( - aborted: boolean, - detail: Pick, - expectedSnapshotId: string, - expectedNodeId: string, -): boolean { - return !aborted && detail.snapshotId === expectedSnapshotId && detail.node.id === expectedNodeId; -} - -export function cacheGraphNodeDetail( - cache: Map, - key: string, - detail: GraphNodeDetail, - limit = 128, -): void { - cache.delete(key); - cache.set(key, detail); - while (cache.size > Math.max(1, limit)) { - const oldest = cache.keys().next().value as string | undefined; - if (oldest === undefined) break; - cache.delete(oldest); - } -} - -export function graphWithNodeNeighborhood(graph: GraphVisualization, detail: GraphNodeDetail): GraphVisualization { - if (graph.mode !== 'detail') return graph; - const nodesById = new Map(graph.nodes.slice(0, MANAGER_GRAPH_MAX_NODE_LIMIT).map(node => [node.id, node])); - const existingRoot = nodesById.get(detail.node.id); - if (existingRoot || nodesById.size < MANAGER_GRAPH_MAX_NODE_LIMIT) { - nodesById.set(detail.node.id, { - ...existingRoot, - degree: existingRoot?.degree ?? 0, - exported: detail.node.exported, - id: detail.node.id, - kind: detail.node.kind, - label: detail.node.label, - language: detail.node.language, - packageName: detail.node.packageName, - path: detail.node.path, - projectId: detail.node.projectId, - qualifiedName: detail.node.qualifiedName, - signature: detail.node.signature, - type: 'symbol', - }); - } - - const edgesById = new Map(graph.edges.slice(0, MANAGER_GRAPH_MAX_EDGE_LIMIT).map(edge => [edge.id, edge])); - let truncated = graph.nodes.length > nodesById.size || graph.edges.length > edgesById.size; - for (const relationship of detail.relationships.slice(0, MAX_EXPANDED_NEIGHBOR_EDGES)) { - const relatedId = relationship.related.id; - if (!relatedId || relatedId === detail.node.id) continue; - if (!nodesById.has(relatedId)) { - if (nodesById.size >= MANAGER_GRAPH_MAX_NODE_LIMIT) { - truncated = true; - continue; - } - nodesById.set(relatedId, { - degree: 0, - id: relatedId, - kind: relationship.related.kind ?? 'symbol', - label: relationship.related.label, - path: relationship.related.path, - projectId: relationship.related.projectId ?? detail.node.projectId, - qualifiedName: relationship.related.qualifiedName, - type: 'symbol', - }); - } - if (!edgesById.has(relationship.id)) { - if (edgesById.size >= MANAGER_GRAPH_MAX_EDGE_LIMIT || !nodesById.has(detail.node.id)) { - truncated = true; - continue; - } - const outgoing = relationship.direction === 'outgoing'; - edgesById.set(relationship.id, { - confidence: relationship.confidence, - count: 1, - id: relationship.id, - provenance: relationship.provenance, - relation: relationship.relation, - sourceId: outgoing ? detail.node.id : relatedId, - targetId: outgoing ? relatedId : detail.node.id, - }); - } - } - - const edges = [...edgesById.values()]; - const degrees = graphNodeSizeValues(edges, 'connections'); - const nodes = [...nodesById.values()].map(node => ({...node, degree: degrees.get(node.id) ?? 0})); - const addedNodes = nodes.length - graph.nodes.length; - return { - ...graph, - edges, - nodes, - stats: { - ...graph.stats, - renderedEdges: edges.length, - renderedNodes: nodes.length, - }, - paging: {...graph.paging, hasMore: graph.paging.hasMore || truncated || detail.stats.truncated}, - warnings: [ - ...graph.warnings, - ...(addedNodes > 0 ? [`Loaded ${addedNodes.toLocaleString()} direct neighbors for ${detail.node.label}.`] : []), - ...(truncated ? ['Direct-neighbor expansion reached the global Manager graph budget.'] : []), - ], - }; -} - -export interface GraphVisualization { - readonly edges: readonly GraphEdge[]; - readonly mode: 'detail' | 'overview'; - readonly nodes: readonly GraphNode[]; - readonly paging: { - readonly edgeLimit: number; - readonly hasMore: boolean; - readonly nodeLimit: number; - }; - readonly projectId: string; - readonly query?: GraphQueryMetadata; - readonly repository: Pick; - readonly scope: {readonly id: string; readonly label: string}; - readonly stats: { - readonly renderedEdges: number; - readonly renderedNodes: number; - readonly totalEdges: number; - readonly totalNodes: number; - }; - readonly warnings: readonly string[]; -} - -export interface GraphQueryMetadata { - readonly matchedNodes: number; - readonly state: 'ready'; - readonly text: string; - readonly warnings: readonly string[]; -} - -export interface GraphQueryVisualization extends GraphVisualization { - readonly query: GraphQueryMetadata; -} - -interface GraphCatalogContinuation { - readonly projectOffset: number; - readonly projectHasMore: boolean; - readonly viewOffset: number; - readonly viewHasMore: boolean; - readonly viewId: string; - readonly workspaceOffset: number; - readonly workspaceHasMore: boolean; -} - -export interface GraphCatalogSearchOptions { - readonly projects: readonly { - readonly description: string; - readonly id: string; - readonly label: string; - readonly viewId: string; - }[]; - readonly views: readonly { - readonly description: string; - readonly id: string; - readonly label: string; - readonly repositoryId: string; - }[]; -} - -export function graphCatalogSearchOptions( - repository: GraphRepository, - repositories: readonly GraphRepositoryGroup[], -): GraphCatalogSearchOptions { - const workspaces = new Map(repository.workspaces.map(workspace => [workspace.id, workspace])); - const projects = repository.projects - .map(project => { - const workspace = project.workspaceId ? workspaces.get(project.workspaceId) : undefined; - return { - description: [workspace?.name, graphProjectBadge(project)].filter(Boolean).join(' · '), - id: project.id, - label: project.label, - viewId: repository.id, - }; - }) - .sort((left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id)); - const viewsById = new Map< - string, - {readonly description: string; readonly id: string; readonly label: string; readonly repositoryId: string} - >(); - for (const group of repositories) { - for (const view of group.views) { - viewsById.set(view.id, { - description: `${group.displayName} · ${view.snapshot.commit.slice(0, 8)}${view.snapshot.dirty ? ' · dirty' : ''} · folder ${graphLocalAssociationText(view.localAssociation)}`, - id: view.id, - label: view.label, - repositoryId: group.id, - }); - } - } - return { - projects, - views: [...viewsById.values()].sort( - (left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id), - ), - }; -} - -export function graphCatalogContinuationHasMore( - continuation: GraphCatalogContinuation | undefined, - viewId: string | undefined, - field: 'projectHasMore' | 'viewHasMore' | 'workspaceHasMore', - fallback: boolean, -): boolean { - return continuation !== undefined && continuation.viewId === viewId ? continuation[field] : fallback; -} - -export interface GraphAnalysis { - readonly communities: readonly { - readonly id: string; - readonly label: string; - readonly memberCount: number; - }[]; - readonly coverage: { - readonly complete: boolean; - readonly topology: { - readonly complete: boolean; - readonly state: 'complete' | 'not-requested' | 'partial' | 'unavailable'; - }; - }; - readonly hubs: readonly { - readonly classification: 'god-node' | 'hub'; - readonly degree: number; - readonly node: {readonly label: string; readonly path: string}; - }[]; - readonly statistics: { - readonly analyzedEdgeCount: number; - readonly analyzedNodeCount: number; - readonly communityCount: number; - readonly connectedComponentCount: number; - readonly maximumDegree: number; - }; - readonly surprisingLinks: readonly { - readonly relation: string; - readonly score: number; - readonly source: {readonly label: string}; - readonly target: {readonly label: string}; - }[]; - readonly warnings: readonly string[]; -} - -export function graphAnalysisTopologyAvailable(analysis: GraphAnalysis): boolean { - return analysis.coverage.topology.state === 'complete' || analysis.coverage.topology.state === 'partial'; -} - -export function graphAnalysisCoverageLabel(analysis: GraphAnalysis): string { - switch (analysis.coverage.topology.state) { - case 'complete': - return analysis.coverage.complete ? 'Complete' : 'Topology complete'; - case 'partial': - return 'Topology partial'; - case 'not-requested': - return 'Topology not requested'; - case 'unavailable': - return 'Topology unavailable'; - } -} - -interface PositionedNode extends GraphNode { - readonly color: THREE.Color; - readonly radius: number; - readonly x: number; - readonly y: number; -} - -interface GraphLayout { - readonly bounds: {readonly height: number; readonly width: number}; - readonly nodes: readonly PositionedNode[]; - readonly nodesById: ReadonlyMap; -} - -export interface GraphPosition { - readonly x: number; - readonly y: number; -} - -interface GraphLabelSize { - readonly height: number; - readonly width: number; -} - -interface GraphRuntime { - readonly camera: THREE.OrthographicCamera; - readonly edgePosition: THREE.BufferAttribute; - readonly edges: readonly GraphEdge[]; - readonly highlightPosition?: THREE.BufferAttribute; - readonly highlightedEdges: readonly GraphEdge[]; - readonly nodeIds: readonly string[]; - readonly nodePosition: THREE.BufferAttribute; - readonly pointMaterials: readonly THREE.ShaderMaterial[]; - readonly renderer: THREE.WebGLRenderer; - readonly scene: THREE.Scene; - readonly selectedNodeId?: string; - readonly selectedPosition?: THREE.BufferAttribute; -} - -export interface ViewState { - readonly x: number; - readonly y: number; - readonly zoom: number; -} - -export type GraphFocusMode = 'all' | 'incoming' | 'neighbors' | 'outgoing'; -export type GraphSizeMetric = 'connections' | 'incoming' | 'outgoing'; - -const GRAPH_PALETTE = ['#67e8c7', '#7aa2ff', '#c08cff', '#ff9f7a', '#f7d56b', '#75d8ff', '#ef88b7', '#9be27d']; -const SELECTED_NODE_COLOR = '#ff4fd8'; -const MIN_ZOOM = 0.32; -const MAX_ZOOM = 8; -const DEFAULT_WORKING_SET = { - edgeLimit: MANAGER_GRAPH_DEFAULT_EDGE_LIMIT, - nodeLimit: MANAGER_GRAPH_DEFAULT_NODE_LIMIT, -} as const; -const MAX_WORKING_SET = { - edgeLimit: MANAGER_GRAPH_MAX_EDGE_LIMIT, - nodeLimit: MANAGER_GRAPH_MAX_NODE_LIMIT, -} as const; -const MAX_ANIMATED_NEIGHBOR_EDGES = 120; -const MAX_EXPANDED_NEIGHBOR_EDGES = 160; -const MAX_FOCUSED_LABELS = 24; -const FOCUS_LAYOUT_ZOOM = 2.8; -const SEARCH_FOCUS_ZOOM = { - detail: 2.8, - overview: 1.8, -} as const; -const GRAPH_QUERY_DEBOUNCE_MILLISECONDS = 450; -const GRAPH_QUERY_MINIMUM_LENGTH = 3; -const GRAPH_QUERY_MAXIMUM_LENGTH = 512; -const DEFAULT_QUERY_WORKING_SET = {edgeLimit: 240, nodeLimit: 120} as const; -const MAX_QUERY_WORKING_SET = {edgeLimit: 500, nodeLimit: 200} as const; - -export function managerGraphQueryCandidate(input: string): string | undefined { - const candidate = input.trim(); - return candidate.length > 0 && candidate.length <= GRAPH_QUERY_MAXIMUM_LENGTH ? candidate : undefined; -} - -export function managerGraphDebouncedQueryCandidate(input: string): string | undefined { - const candidate = managerGraphQueryCandidate(input); - return candidate && candidate.length >= GRAPH_QUERY_MINIMUM_LENGTH ? candidate : undefined; -} - -export function managerGraphClientRenderProxy( - graph: GraphVisualization, - size: {readonly height: number; readonly width: number} = {height: 720, width: 1_280}, -): {readonly labels: number; readonly matchedEdges: number; readonly nodes: number} { - const layout = buildGraphLayout(graph, 'connections', graph.edges); - const view = fittedView(layout, size); - let matchedEdges = 0; - for (const edge of graph.edges) { - if (layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId)) matchedEdges += 1; - } - return { - labels: visibleLabels(layout, graph.mode, size, view).length, - matchedEdges, - nodes: layout.nodes.length, - }; -} - -export function graphOverviewSizeLabel(graph: GraphVisualization): string { - return graph.repository.metrics === 'complete' && graph.nodes.some(node => node.symbolCount !== undefined) - ? 'Component symbols' - : 'Visible relationship degree'; -} - -export function GraphWorkspace(props: { - readonly administration?: CodeGraphLocalDiagnosticsReport; - readonly administrationBusy?: string; - readonly administrationOutput?: string; - readonly catalog?: GraphCatalog; - readonly catalogError?: string; - readonly loadAnalysis: (repositoryId: string, snapshotId: string, signal: AbortSignal) => Promise; - readonly loadGraph: ( - repositoryId: string, - snapshotId: string, - projectId: string, - limits: ManagerGraphVisualizationLimits, - signal: AbortSignal, - ) => Promise; - readonly loadCatalogPage: ( - repositoryId: string, - snapshotId: string, - projectOffset: number, - workspaceOffset: number, - query: string, - signal: AbortSignal, - ) => Promise; - readonly loadNodeDetail: ( - repositoryId: string, - snapshotId: string, - nodeId: string, - signal: AbortSignal, - ) => Promise; - readonly loadQuery: ( - repositoryId: string, - snapshotId: string, - query: string, - limits: ManagerGraphVisualizationLimits, - signal: AbortSignal, - ) => Promise; - readonly loadViewsPage: ( - repositoryId: string, - offset: number, - query: string, - signal: AbortSignal, - ) => Promise; - readonly onAdministrationAction?: (action: GraphAdministrationAction) => void; - readonly onDiagnostics?: (options: {readonly analyze: boolean; readonly deep: boolean}) => void; - readonly onRefresh: () => void; -}): React.ReactElement { - const [repositoryId, setRepositoryId] = useState(''); - const [viewId, setViewId] = useState(''); - const [projectId, setProjectId] = useState('all'); - const [baseGraph, setBaseGraph] = useState(); - const [workingSet, setWorkingSet] = useState(DEFAULT_WORKING_SET); - const [expandedNeighborhood, setExpandedNeighborhood] = useState(); - const [selectedNodeId, setSelectedNodeId] = useState(); - const [focusRequest, setFocusRequest] = useState<{readonly nodeId: string; readonly sequence: number} | undefined>(); - const focusSequence = useRef(0); - const [search, setSearch] = useState(''); - const [queryInput, setQueryInput] = useState(''); - const [activeQuery, setActiveQuery] = useState(''); - const [queryGraph, setQueryGraph] = useState(); - const [queryLoading, setQueryLoading] = useState(false); - const [queryError, setQueryError] = useState(''); - const [queryAttempt, setQueryAttempt] = useState(0); - const [queryWorkingSet, setQueryWorkingSet] = useState(DEFAULT_QUERY_WORKING_SET); - const queryRequestGate = useRef(createGraphQueryRequestGate()); - const [relationFilter, setRelationFilter] = useState('all'); - const [focusMode, setFocusMode] = useState('all'); - const [sizeMetric, setSizeMetric] = useState('connections'); - const [nodeDetail, setNodeDetail] = useState(); - const [nodeDetailLoading, setNodeDetailLoading] = useState(false); - const [nodeDetailError, setNodeDetailError] = useState(''); - const nodeDetailCache = useRef(new Map()); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(''); - const [analysis, setAnalysis] = useState(); - const [analysisLoading, setAnalysisLoading] = useState(false); - const [analysisError, setAnalysisError] = useState(''); - const analysisRequestSequence = useRef(0); - const analysisAbortController = useRef(undefined); - const graphRequestSequence = useRef(0); - const [catalogAdditions, setCatalogAdditions] = useState([]); - const [catalogQuery, setCatalogQuery] = useState(''); - const [catalogLoading, setCatalogLoading] = useState(false); - const [catalogError, setCatalogError] = useState(''); - const [catalogSearchResult, setCatalogSearchResult] = useState< - {readonly options: GraphCatalogSearchOptions; readonly query: string} | undefined - >(); - const [catalogContinuation, setCatalogContinuation] = useState(); - const catalogAbortController = useRef(undefined); - const catalogRequestSequence = useRef(0); - const baseCatalogIdentity = useMemo( - () => - (props.catalog?.repositories ?? []) - .flatMap(group => group.views.map(view => `${view.id}:${view.snapshot.id}`)) - .sort(compareCodeUnits) - .join('|'), - [props.catalog?.repositories], - ); - const repositories = useMemo( - () => mergeGraphRepositoryGroups(props.catalog?.repositories ?? [], catalogAdditions), - [catalogAdditions, props.catalog?.repositories], - ); - const repositoryGroup = repositories.find(candidate => candidate.id === repositoryId) ?? repositories[0]; - const repository = - repositoryGroup?.views.find(candidate => candidate.id === viewId) ?? - repositoryGroup?.views.find(candidate => candidate.id === repositoryGroup.defaultViewId) ?? - repositoryGroup?.views[0]; - const baseRepositoryGroup = (props.catalog?.repositories ?? []).find( - candidate => candidate.id === repositoryGroup?.id, - ); - const baseRepository = baseRepositoryGroup?.views.find(candidate => candidate.id === repository?.id); - const analysisScope = `${repository?.id ?? ''}:${repository?.snapshot.id ?? ''}`; - const analysisScopeRef = useRef(analysisScope); - analysisScopeRef.current = analysisScope; - const graphScope = `${analysisScope}:${projectId}:${workingSet.nodeLimit}:${workingSet.edgeLimit}`; - const graphScopeRef = useRef(graphScope); - graphScopeRef.current = graphScope; - const queryScope = `${analysisScope}:${activeQuery}:${queryAttempt}:${queryWorkingSet.nodeLimit}:${queryWorkingSet.edgeLimit}`; - const graphSource = activeQuery ? queryGraph : baseGraph; - const graph = useMemo( - () => - graphSource && expandedNeighborhood ? graphWithNodeNeighborhood(graphSource, expandedNeighborhood) : graphSource, - [expandedNeighborhood, graphSource], - ); - const selectedNode = graph?.nodes.find(node => node.id === selectedNodeId); - const relations = useMemo( - () => [...new Set(graph?.edges.map(edge => edge.relation) ?? [])].sort(compareCodeUnits), - [graph], - ); - const activeBuilds = (props.catalog?.builds ?? []).filter(graphBuildShouldDisplay); - const selectedRepositoryIsIndexing = activeBuilds.some( - build => - repository !== undefined && - build.identity.checkoutId === repository.checkoutId && - build.identity.worktreeId === repository.worktreeId && - (build.state === 'queued' || build.state === 'running'), - ); - const workingSetAtMaximum = activeQuery - ? queryWorkingSet.nodeLimit >= MAX_QUERY_WORKING_SET.nodeLimit && - queryWorkingSet.edgeLimit >= MAX_QUERY_WORKING_SET.edgeLimit - : workingSet.nodeLimit >= MAX_WORKING_SET.nodeLimit && workingSet.edgeLimit >= MAX_WORKING_SET.edgeLimit; - const projectCatalogHasMore = graphCatalogContinuationHasMore( - catalogContinuation, - repository?.id, - 'projectHasMore', - repository?.projectsTruncated ?? false, - ); - const workspaceCatalogHasMore = graphCatalogContinuationHasMore( - catalogContinuation, - repository?.id, - 'workspaceHasMore', - repository?.workspacesTruncated ?? false, - ); - const viewCatalogHasMore = graphCatalogContinuationHasMore( - catalogContinuation, - repository?.id, - 'viewHasMore', - repositoryGroup?.viewsTruncated ?? false, - ); - - useEffect(() => { - const selection = resolveGraphSelection(repositories, repositoryId, viewId); - if (selection.repositoryId !== repositoryId) { - setRepositoryId(selection.repositoryId); - setProjectId('all'); - setWorkingSet(DEFAULT_WORKING_SET); - } - if (selection.viewId !== viewId) { - setViewId(selection.viewId); - setProjectId('all'); - setWorkingSet(DEFAULT_WORKING_SET); - } - }, [repositories, repositoryId, viewId]); - - useEffect(() => { - if (!repository) { - setBaseGraph(undefined); - setExpandedNeighborhood(undefined); - return; - } - const requestSequence = graphRequestSequence.current + 1; - graphRequestSequence.current = requestSequence; - const requestedScope = graphScope; - const controller = new AbortController(); - setLoading(true); - setError(''); - setSelectedNodeId(undefined); - setExpandedNeighborhood(undefined); - setFocusRequest(undefined); - setFocusMode('all'); - setRelationFilter('all'); - setSizeMetric('connections'); - void props - .loadGraph(repository.id, repository.snapshot.id, projectId, workingSet, controller.signal) - .then(next => { - if ( - graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) - ) { - setBaseGraph(next); - } - }) - .catch(cause => { - if ( - !isAbortError(cause) && - graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) - ) { - setBaseGraph(undefined); - setError(cause instanceof Error ? cause.message : String(cause)); - } - }) - .finally(() => { - if ( - graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) - ) { - setLoading(false); - } - }); - return () => { - controller.abort(); - graphRequestSequence.current += 1; - }; - }, [graphScope, projectId, props.loadGraph, repository?.id, repository?.snapshot.id, workingSet]); - - useEffect(() => { - const candidate = managerGraphDebouncedQueryCandidate(queryInput); - if (!candidate || candidate === activeQuery) return; - const timeout = window.setTimeout(() => { - setQueryAttempt(0); - setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); - setActiveQuery(candidate); - }, GRAPH_QUERY_DEBOUNCE_MILLISECONDS); - return () => window.clearTimeout(timeout); - }, [activeQuery, queryInput]); - - useEffect(() => { - if (!repository || !activeQuery) { - setQueryGraph(undefined); - setQueryLoading(false); - setQueryError(''); - return; - } - const expectedSnapshotId = repository.snapshot.id; - const expectedQuery = activeQuery; - const request = queryRequestGate.current.request({expectedQuery, expectedSnapshotId, scope: queryScope}, signal => - props.loadQuery(repository.id, expectedSnapshotId, expectedQuery, queryWorkingSet, signal), - ); - setQueryGraph(undefined); - setQueryLoading(true); - setQueryError(''); - setSelectedNodeId(undefined); - setExpandedNeighborhood(undefined); - setFocusRequest(undefined); - setFocusMode('all'); - setRelationFilter('all'); - setSizeMetric('connections'); - void request.result.then(outcome => { - if (!request.isCurrent()) return; - if (outcome.state === 'accepted') { - setQueryGraph(outcome.graph); - } else if (outcome.state === 'failed') { - setQueryGraph(undefined); - setQueryError(outcome.cause instanceof Error ? outcome.cause.message : String(outcome.cause)); - } - setQueryLoading(false); - }); - return () => { - request.cancel(); - }; - }, [activeQuery, props.loadQuery, queryScope, queryWorkingSet, repository?.id, repository?.snapshot.id]); - - useEffect(() => { - analysisAbortController.current?.abort(); - analysisRequestSequence.current += 1; - setAnalysis(undefined); - setAnalysisError(''); - setAnalysisLoading(false); - return () => { - analysisAbortController.current?.abort(); - analysisRequestSequence.current += 1; - }; - }, [repository?.id, repository?.snapshot.id]); - - const loadAnalysis = (): void => { - if (!repository || analysisLoading) return; - const requestedScope = analysisScope; - const requestSequence = analysisRequestSequence.current + 1; - analysisRequestSequence.current = requestSequence; - analysisAbortController.current?.abort(); - const controller = new AbortController(); - analysisAbortController.current = controller; - setAnalysisLoading(true); - setAnalysisError(''); - void props - .loadAnalysis(repository.id, repository.snapshot.id, controller.signal) - .then(next => { - if ( - graphAnalysisRequestIsCurrent( - analysisRequestSequence.current, - requestSequence, - analysisScopeRef.current, - requestedScope, - ) - ) { - setAnalysis(next); - } - }) - .catch(cause => { - if ( - !isAbortError(cause) && - graphAnalysisRequestIsCurrent( - analysisRequestSequence.current, - requestSequence, - analysisScopeRef.current, - requestedScope, - ) - ) { - setAnalysisError(cause instanceof Error ? cause.message : String(cause)); - } - }) - .finally(() => { - if ( - graphAnalysisRequestIsCurrent( - analysisRequestSequence.current, - requestSequence, - analysisScopeRef.current, - requestedScope, - ) - ) { - setAnalysisLoading(false); - } - }); - }; - - useEffect(() => { - if (!selectedNode || selectedNode.type !== 'symbol' || !repository) { - setNodeDetail(undefined); - setNodeDetailLoading(false); - setNodeDetailError(''); - return; - } - const key = `${repository.id}:${graph?.repository.snapshot.id ?? ''}:${selectedNode.id}`; - const cached = nodeDetailCache.current.get(key); - if (cached) { - cacheGraphNodeDetail(nodeDetailCache.current, key, cached); - setNodeDetail(cached); - setExpandedNeighborhood(cached); - setNodeDetailLoading(false); - setNodeDetailError(''); - focusSequence.current += 1; - setFocusRequest({nodeId: cached.node.id, sequence: focusSequence.current}); - return; - } - const controller = new AbortController(); - setNodeDetail(undefined); - setNodeDetailLoading(true); - setNodeDetailError(''); - void props - .loadNodeDetail(repository.id, repository.snapshot.id, selectedNode.id, controller.signal) - .then(detail => { - if ( - !graphNodeDetailRequestIsCurrent(controller.signal.aborted, detail, repository.snapshot.id, selectedNode.id) - ) - return; - cacheGraphNodeDetail(nodeDetailCache.current, key, detail); - setNodeDetail(detail); - setExpandedNeighborhood(detail); - focusSequence.current += 1; - setFocusRequest({nodeId: detail.node.id, sequence: focusSequence.current}); - }) - .catch(cause => { - if (!controller.signal.aborted && !isAbortError(cause)) { - setExpandedNeighborhood(undefined); - setNodeDetailError(cause instanceof Error ? cause.message : String(cause)); - } - }) - .finally(() => { - if (!controller.signal.aborted) setNodeDetailLoading(false); - }); - return () => { - controller.abort(); - }; - }, [graph?.repository.snapshot.id, props.loadNodeDetail, repository?.id, selectedNode?.id, selectedNode?.type]); - - const searchResults = useMemo(() => { - const needle = search.trim().toLowerCase(); - if (!needle || !graph) return []; - return graph.nodes - .filter( - node => - node.label.toLowerCase().includes(needle) || - node.qualifiedName?.toLowerCase().includes(needle) || - node.path?.toLowerCase().includes(needle), - ) - .sort((left, right) => right.degree - left.degree || compareCodeUnits(left.label, right.label)) - .slice(0, 8); - }, [graph, search]); - - const chooseRepository = (nextRepositoryId: string): void => { - const next = repositories.find(candidate => candidate.id === nextRepositoryId); - setRepositoryId(nextRepositoryId); - setViewId(next?.defaultViewId ?? next?.views[0]?.id ?? ''); - setProjectId('all'); - setWorkingSet(DEFAULT_WORKING_SET); - clearCatalogSearch(); - clearCodeQuery(); - }; - - const chooseView = (nextViewId: string): void => { - setViewId(nextViewId); - setProjectId('all'); - setWorkingSet(DEFAULT_WORKING_SET); - clearCatalogSearch(); - clearCodeQuery(); - }; - - const chooseCatalogView = (nextRepositoryId: string, nextViewId: string): void => { - setRepositoryId(nextRepositoryId); - setViewId(nextViewId); - setProjectId('all'); - setWorkingSet(DEFAULT_WORKING_SET); - clearCatalogSearch(); - clearCodeQuery(); - }; - - const chooseProject = (nextProjectId: string): void => { - setProjectId(nextProjectId); - setWorkingSet(DEFAULT_WORKING_SET); - setSearch(''); - setSelectedNodeId(undefined); - setExpandedNeighborhood(undefined); - clearCatalogSearch(); - clearCodeQuery(); - }; - - function clearCatalogSearch(): void { - setCatalogQuery(''); - setCatalogSearchResult(undefined); - setCatalogError(''); - } - - const submitCodeQuery = (): void => { - const candidate = managerGraphQueryCandidate(queryInput); - if (!candidate) { - setQueryError(`Enter between 1 and ${GRAPH_QUERY_MAXIMUM_LENGTH} characters to search the code graph.`); - return; - } - if (candidate === activeQuery) { - setQueryAttempt(current => current + 1); - return; - } - setQueryAttempt(0); - setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); - setActiveQuery(candidate); - }; - - function clearCodeQuery(): void { - queryRequestGate.current.cancelCurrent(); - setQueryInput(''); - setActiveQuery(''); - setQueryGraph(undefined); - setQueryLoading(false); - setQueryError(''); - setQueryAttempt(0); - setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); - setSelectedNodeId(undefined); - setExpandedNeighborhood(undefined); - setFocusRequest(undefined); - } - - const selectNode = (nodeId: string | undefined, focus = false): void => { - setSelectedNodeId(nodeId); - if (!nodeId) { - setFocusMode('all'); - setExpandedNeighborhood(undefined); - return; - } - if (baseGraph?.nodes.some(node => node.id === nodeId)) setExpandedNeighborhood(undefined); - if (focus) { - focusSequence.current += 1; - setFocusRequest({nodeId, sequence: focusSequence.current}); - } - }; - - const loadCatalogContinuation = (requestedQuery: string): void => { - if (!repository || !repositoryGroup || catalogLoading) return; - const query = requestedQuery.trim().slice(0, 256); - const continuation = catalogContinuation?.viewId === repository.id ? catalogContinuation : undefined; - const offsets = - query.length === 0 - ? graphCatalogPageOffsets({ - baseRepository, - baseRepositoryGroup, - checkoutId: repository.checkoutId, - continuation, - viewId: repository.id, - }) - : {projectOffset: 0, viewOffset: 0, workspaceOffset: 0}; - const {projectOffset, viewOffset, workspaceOffset} = offsets; - const requestedScope = `${repository.id}:${repository.snapshot.id}:${projectOffset}:${workspaceOffset}:${viewOffset}:${query}`; - const requestSequence = catalogRequestSequence.current + 1; - catalogRequestSequence.current = requestSequence; - catalogAbortController.current?.abort(); - const controller = new AbortController(); - catalogAbortController.current = controller; - setCatalogLoading(true); - setCatalogError(''); - void Promise.all([ - props.loadCatalogPage( - repository.id, - repository.snapshot.id, - projectOffset, - workspaceOffset, - query, - controller.signal, - ), - props.loadViewsPage(repository.id, viewOffset, query, controller.signal), - ]) - .then(([catalogPage, viewPage]) => { - const currentScope = `${repository.id}:${repository.snapshot.id}:${projectOffset}:${workspaceOffset}:${viewOffset}:${query}`; - if ( - controller.signal.aborted || - catalogRequestSequence.current !== requestSequence || - currentScope !== requestedScope - ) - return; - const selectedViewGroup: GraphRepositoryGroup = { - ...repositoryGroup, - defaultViewId: repositoryGroup.defaultViewId, - views: [catalogPage.repository], - viewsTruncated: false, - }; - setCatalogAdditions(current => - mergeGraphRepositoryGroups(current, [selectedViewGroup, ...viewPage.repositories]), - ); - if (query.length > 0) { - setCatalogSearchResult({ - options: graphCatalogSearchOptions(catalogPage.repository, viewPage.repositories), - query, - }); - } - if (query.length === 0) { - setCatalogContinuation({ - projectHasMore: catalogPage.repository.projectsTruncated, - projectOffset: - projectOffset + catalogPage.repository.projects.filter(project => project.id.startsWith('cgp_')).length, - viewHasMore: viewPage.hasMore, - viewId: repository.id, - viewOffset: - viewOffset + - viewPage.repositories - .flatMap(group => group.views) - .filter(view => view.checkoutId === repository.checkoutId).length, - workspaceHasMore: catalogPage.repository.workspacesTruncated, - workspaceOffset: workspaceOffset + catalogPage.repository.workspaces.length, - }); - } - }) - .catch(cause => { - if (!controller.signal.aborted && catalogRequestSequence.current === requestSequence) { - setCatalogError(cause instanceof Error ? cause.message : String(cause)); - } - }) - .finally(() => { - if (!controller.signal.aborted && catalogRequestSequence.current === requestSequence) setCatalogLoading(false); - }); - }; - - useEffect(() => { - setCatalogAdditions([]); - }, [baseCatalogIdentity]); - - useEffect(() => { - catalogAbortController.current?.abort(); - catalogRequestSequence.current += 1; - setCatalogContinuation(undefined); - setCatalogError(''); - setCatalogSearchResult(undefined); - setCatalogLoading(false); - }, [baseCatalogIdentity, repository?.id, repository?.snapshot.id]); - - return ( -
-
-
-

Native code intelligence

-

Knowledge graph

-

- Explore architecture from repository-level structure down to individual symbols. -

-
- -
- -
- undefined)} - onDiagnostics={props.onDiagnostics ?? (() => undefined)} - output={props.administrationOutput} - report={props.administration} - /> - {props.catalog?.maintenance ? : null} - {activeBuilds.length > 0 ? ( -
- {activeBuilds.map(build => ( - - ))} -
- ) : null} - - {props.catalog?.diagnostics.length ? ( -
- Some indexed views need attention - {props.catalog.diagnostics.map(diagnostic => ( - {diagnostic.message} - ))} -
- ) : null} -
- -
-
- - {repositoryGroup && (repositoryGroup.views.length > 1 || viewCatalogHasMore) ? ( - - ) : null} - -
-
- -
- { - setCatalogQuery(event.target.value); - setCatalogSearchResult(undefined); - setCatalogError(''); - }} - onKeyDown={event => { - if (event.key !== 'Enter') return; - event.preventDefault(); - loadCatalogContinuation(catalogQuery); - }} - placeholder="Component, workspace, commit, or view" - type="search" - value={catalogQuery} - /> - -
- {projectCatalogHasMore || workspaceCatalogHasMore || viewCatalogHasMore ? ( - - ) : null} - {catalogError ? {catalogError} : null} - {catalogSearchResult ? ( -
- {catalogSearchResult.options.projects.length + catalogSearchResult.options.views.length > 0 ? ( - <> -

- Found{' '} - {( - catalogSearchResult.options.projects.length + catalogSearchResult.options.views.length - ).toLocaleString()}{' '} - options for “{catalogSearchResult.query}” -

- {catalogSearchResult.options.projects.length > 0 ? ( -
- Components and workspace matches - {catalogSearchResult.options.projects.map(option => ( - - ))} -
- ) : null} - {catalogSearchResult.options.views.length > 0 ? ( -
- Indexed views - {catalogSearchResult.options.views.map(option => ( - - ))} -
- ) : null} - - ) : ( -

No catalog matches for “{catalogSearchResult.query}”

- )} -
- ) : ( - - Search results appear here. - - )} -
-
- - setSearch(event.target.value)} - placeholder={graph?.mode === 'overview' ? 'Search components' : 'Name, path, or symbol'} - type="search" - value={search} - /> - {search.trim() ? ( -
- {searchResults.length > 0 ? ( - searchResults.map(node => ( - - )) - ) : ( -

No matching nodes

- )} -
- ) : null} -
-
- -
- setQueryInput(event.target.value)} - onKeyDown={event => { - if (event.key !== 'Enter') return; - event.preventDefault(); - submitCodeQuery(); - }} - placeholder="Concept, path, module, or symbol" - type="search" - value={queryInput} - /> - -
- {activeQuery ? ( - - ) : null} - {!activeQuery && queryError ? {queryError} : null} -
-
- {graph ? compactNumber(graph.stats.renderedNodes) : '—'} nodes - {graph ? compactNumber(graph.stats.renderedEdges) : '—'} links - {graph?.paging.hasMore ? ( - - ) : null} - WebGL -
-
- - {graph ? ( -
- - {graph.mode === 'detail' ? ( - - ) : ( -
- Node size - {graphOverviewSizeLabel(graph)} -
- )} -
- Selection focus -
- {( - [ - ['all', 'All'], - ['neighbors', 'Neighbors'], - ['incoming', 'Incoming'], - ['outgoing', 'Outgoing'], - ] as const - ).map(([mode, label]) => ( - - ))} -
-
- {selectedNode ? ( - - ) : ( -

Select a node to isolate its neighborhood and direction.

- )} -
- ) : null} - -
-
- {!props.catalog && props.catalogError ? ( -
-
- ) : !props.catalog ? ( -
-
- ) : repositories.length === 0 ? ( - build.state === 'queued' || build.state === 'running')} - /> - ) : activeQuery && selectedRepositoryIsIndexing && !queryGraph ? ( -
-
- ) : activeQuery && queryError ? ( -
-
- ) : !activeQuery && error ? ( -
-
- ) : (activeQuery ? queryLoading : loading) || !graph ? ( -
-
- ) : activeQuery && (graph.query?.matchedNodes === 0 || graph.nodes.length === 0) ? ( -
-
- ) : ( - selectNode(nodeId, Boolean(nodeId))} - relationFilter={relationFilter} - sizeMetric={sizeMetric} - focusRequest={focusRequest} - selectedNodeId={selectedNodeId} - /> - )} -
- - -
- - {graph && [...new Set([...graph.warnings, ...(graph.query?.warnings ?? [])])].length ? ( -
- {[...new Set([...graph.warnings, ...(graph.query?.warnings ?? [])])].map(warning => ( - {warning} - ))} -
- ) : null} -
- ); -} - -function ThreeGraph(props: { - readonly focusRequest?: {readonly nodeId: string; readonly sequence: number}; - readonly focusMode: GraphFocusMode; - readonly graph: GraphVisualization; - readonly onOpenProject: (projectId: string) => void; - readonly onSelectNode: (nodeId: string | undefined) => void; - readonly relationFilter: string; - readonly selectedNodeId?: string; - readonly sizeMetric: GraphSizeMetric; -}): React.ReactElement { - const containerRef = useRef(null); - const canvasRef = useRef(null); - const dragRef = useRef<{moved: boolean; pointerId: number; x: number; y: number} | undefined>(undefined); - const labelRefs = useRef(new Map()); - const livePositionsRef = useRef>(new Map()); - const runtimeRef = useRef(undefined); - const [settledPositions, setSettledPositions] = useState>(() => new Map()); - const [size, setSize] = useState({height: 1, width: 1}); - const sizingEdges = useMemo( - () => - props.relationFilter === 'all' - ? props.graph.edges - : props.graph.edges.filter(edge => edge.relation === props.relationFilter), - [props.graph.edges, props.relationFilter], - ); - const baseLayout = useMemo( - () => buildGraphLayout(props.graph, props.sizeMetric, sizingEdges), - [props.graph, props.sizeMetric, sizingEdges], - ); - const layout = useMemo(() => graphLayoutWithPositions(baseLayout, settledPositions), [baseLayout, settledPositions]); - const displayEdges = useMemo( - () => graphDisplayEdges(props.graph.edges, props.selectedNodeId, props.focusMode, props.relationFilter), - [props.focusMode, props.graph.edges, props.relationFilter, props.selectedNodeId], - ); - const neighborhoodEdges = useMemo( - () => - props.selectedNodeId - ? displayEdges.filter(edge => edge.sourceId === props.selectedNodeId || edge.targetId === props.selectedNodeId) - : [], - [displayEdges, props.selectedNodeId], - ); - const animatedNeighborhoodEdges = useMemo( - () => neighborhoodEdges.slice(0, MAX_ANIMATED_NEIGHBOR_EDGES), - [neighborhoodEdges], - ); - const highlightedNodeIds = useMemo( - () => - props.selectedNodeId - ? new Set([props.selectedNodeId, ...animatedNeighborhoodEdges.flatMap(edge => [edge.sourceId, edge.targetId])]) - : undefined, - [animatedNeighborhoodEdges, props.selectedNodeId], - ); - const activeNodeIds = useMemo(() => { - if (!props.selectedNodeId || props.focusMode === 'all') return undefined; - return new Set([props.selectedNodeId, ...displayEdges.flatMap(edge => [edge.sourceId, edge.targetId])]); - }, [displayEdges, props.focusMode, props.selectedNodeId]); - const [view, setView] = useState(() => fittedView(layout, size)); - const viewRef = useRef(view); - const [focusLayoutRevision, setFocusLayoutRevision] = useState(0); - const [renderError, setRenderError] = useState(''); - - useEffect(() => { - setView(fittedView(layout, size)); - }, [props.graph.projectId, props.graph.repository.id, props.graph.repository.snapshot.id, size.height, size.width]); - - useEffect(() => { - viewRef.current = view; - }, [view]); - - useEffect(() => { - const request = props.focusRequest; - const node = request ? layout.nodesById.get(request.nodeId) : undefined; - if (!request || !node) return; - const startedAt = performance.now(); - const duration = 360; - const start = viewRef.current; - const target = graphFocusTarget(start, graphPosition(node, livePositionsRef.current), props.graph.mode); - let frame = 0; - const animate = (now: number): void => { - const progress = Math.min(1, (now - startedAt) / duration); - const eased = 1 - Math.pow(1 - progress, 3); - setView({ - x: lerp(start.x, target.x, eased), - y: lerp(start.y, target.y, eased), - zoom: lerp(start.zoom, target.zoom, eased), - }); - if (progress < 1) frame = window.requestAnimationFrame(animate); - else setFocusLayoutRevision(current => current + 1); - }; - frame = window.requestAnimationFrame(animate); - return () => window.cancelAnimationFrame(frame); - }, [props.focusRequest?.sequence, props.graph.mode]); - - useEffect(() => { - const container = containerRef.current; - if (!container) return; - const observer = new ResizeObserver(entries => { - const bounds = entries[0]?.contentRect; - if (bounds) setSize({height: Math.max(1, bounds.height), width: Math.max(1, bounds.width)}); - }); - observer.observe(container); - return () => observer.disconnect(); - }, []); - - useEffect(() => { - const canvas = canvasRef.current; - if (!canvas) return; - let renderer: THREE.WebGLRenderer; - try { - renderer = new THREE.WebGLRenderer({ - alpha: true, - antialias: true, - canvas, - powerPreference: 'high-performance', - }); - setRenderError(''); - } catch { - setRenderError('WebGL is unavailable in this browser. Enable hardware acceleration to render the graph.'); - return; - } - renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); - renderer.setSize(size.width, size.height, false); - renderer.outputColorSpace = THREE.SRGBColorSpace; - const scene = new THREE.Scene(); - const camera = new THREE.OrthographicCamera(); - updateCamera(camera, view, size); - const currentPositions = livePositionsRef.current; - - const edgePositions: number[] = []; - const edgeColors: number[] = []; - const renderedEdges = displayEdges.filter( - edge => layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId), - ); - for (const edge of renderedEdges) { - const source = layout.nodesById.get(edge.sourceId); - const target = layout.nodesById.get(edge.targetId); - if (!source || !target) continue; - const sourcePosition = graphPosition(source, currentPositions); - const targetPosition = graphPosition(target, currentPositions); - edgePositions.push(sourcePosition.x, sourcePosition.y, 0, targetPosition.x, targetPosition.y, 0); - edgeColors.push(source.color.r, source.color.g, source.color.b, target.color.r, target.color.g, target.color.b); - } - const edgeGeometry = new THREE.BufferGeometry(); - const edgePosition = new THREE.Float32BufferAttribute(edgePositions, 3); - edgeGeometry.setAttribute('position', edgePosition); - edgeGeometry.setAttribute('color', new THREE.Float32BufferAttribute(edgeColors, 3)); - const edgeMaterial = new THREE.LineBasicMaterial({ - blending: THREE.AdditiveBlending, - opacity: props.graph.mode === 'overview' ? 0.34 : 0.18, - transparent: true, - vertexColors: true, - }); - const lines = new THREE.LineSegments(edgeGeometry, edgeMaterial); - scene.add(lines); - - const positions: number[] = []; - const colors: number[] = []; - const pointSizes: number[] = []; - for (const node of layout.nodes) { - const color = activeNodeIds && !activeNodeIds.has(node.id) ? node.color.clone().multiplyScalar(0.12) : node.color; - const position = graphPosition(node, currentPositions); - positions.push(position.x, position.y, 1); - colors.push(color.r, color.g, color.b); - pointSizes.push(node.radius * 2); - } - const nodeGeometry = new THREE.BufferGeometry(); - const nodePosition = new THREE.Float32BufferAttribute(positions, 3); - nodeGeometry.setAttribute('position', nodePosition); - nodeGeometry.setAttribute('color', new THREE.Float32BufferAttribute(colors, 3)); - nodeGeometry.setAttribute('pointSize', new THREE.Float32BufferAttribute(pointSizes, 1)); - const nodeMaterial = graphPointMaterial(1, view.zoom); - const points = new THREE.Points(nodeGeometry, nodeMaterial); - scene.add(points); - - const renderedHighlightedEdges = animatedNeighborhoodEdges.filter( - edge => layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId), - ); - const highlightPositions = directionalEdgePositions(renderedHighlightedEdges, layout.nodesById, currentPositions); - let highlightGeometry: THREE.BufferGeometry | undefined; - let highlightPosition: THREE.BufferAttribute | undefined; - let highlightMaterial: THREE.LineBasicMaterial | undefined; - if (highlightPositions.length > 0) { - highlightGeometry = new THREE.BufferGeometry(); - highlightPosition = new THREE.Float32BufferAttribute(highlightPositions, 3); - highlightGeometry.setAttribute('position', highlightPosition); - highlightMaterial = new THREE.LineBasicMaterial({ - blending: THREE.AdditiveBlending, - color: SELECTED_NODE_COLOR, - opacity: 0.72, - transparent: true, - }); - scene.add(new THREE.LineSegments(highlightGeometry, highlightMaterial)); - } - - const selectedNode = props.selectedNodeId ? layout.nodesById.get(props.selectedNodeId) : undefined; - let selectedGeometry: THREE.BufferGeometry | undefined; - let selectedPosition: THREE.BufferAttribute | undefined; - let selectedMaterial: THREE.ShaderMaterial | undefined; - if (selectedNode) { - const position = graphPosition(selectedNode, currentPositions); - selectedGeometry = new THREE.BufferGeometry(); - selectedPosition = new THREE.Float32BufferAttribute([position.x, position.y, 2], 3); - selectedGeometry.setAttribute('position', selectedPosition); - selectedGeometry.setAttribute( - 'color', - new THREE.Float32BufferAttribute(new THREE.Color(SELECTED_NODE_COLOR).toArray(), 3), - ); - selectedGeometry.setAttribute('pointSize', new THREE.Float32BufferAttribute([selectedNode.radius * 3.3], 1)); - selectedMaterial = graphPointMaterial(1.3, view.zoom); - scene.add(new THREE.Points(selectedGeometry, selectedMaterial)); - } - - runtimeRef.current = { - camera, - edgePosition, - edges: renderedEdges, - highlightedEdges: renderedHighlightedEdges, - highlightPosition, - nodeIds: layout.nodes.map(node => node.id), - nodePosition, - pointMaterials: selectedMaterial ? [nodeMaterial, selectedMaterial] : [nodeMaterial], - renderer, - scene, - selectedNodeId: selectedNode?.id, - selectedPosition, - }; - renderer.render(scene, camera); - return () => { - runtimeRef.current = undefined; - edgeGeometry.dispose(); - edgeMaterial.dispose(); - nodeGeometry.dispose(); - nodeMaterial.dispose(); - highlightGeometry?.dispose(); - highlightMaterial?.dispose(); - selectedGeometry?.dispose(); - selectedMaterial?.dispose(); - renderer.dispose(); - }; - }, [activeNodeIds, animatedNeighborhoodEdges, displayEdges, layout, props.graph.mode, props.selectedNodeId]); - - useEffect(() => { - const runtime = runtimeRef.current; - if (!runtime) return; - runtime.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); - runtime.renderer.setSize(size.width, size.height, false); - for (const material of runtime.pointMaterials) { - const scale = material.uniforms.viewScale; - if (scale) scale.value = graphPointViewScale(view.zoom); - } - updateCamera(runtime.camera, view, size); - runtime.renderer.render(runtime.scene, runtime.camera); - }, [size, view]); - - useEffect(() => { - const currentNodes = baseLayout.nodes.map(node => { - const settledNode = layout.nodesById.get(node.id) ?? node; - const position = graphPosition(settledNode, livePositionsRef.current); - return {...node, x: position.x, y: position.y}; - }); - const labelSizes = new Map(); - for (const [nodeId, element] of labelRefs.current) { - labelSizes.set(nodeId, {height: element.offsetHeight, width: element.offsetWidth}); - } - const targets = graphFocusLayoutTargets( - currentNodes, - props.selectedNodeId, - animatedNeighborhoodEdges, - labelSizes, - Math.max(FOCUS_LAYOUT_ZOOM, viewRef.current.zoom), - ); - const simulationIds = new Set([...livePositionsRef.current.keys(), ...settledPositions.keys(), ...targets.keys()]); - const particles = [...simulationIds].flatMap(nodeId => { - const baseNode = baseLayout.nodesById.get(nodeId); - const currentNode = layout.nodesById.get(nodeId) ?? baseNode; - if (!baseNode || !currentNode) return []; - const start = livePositionsRef.current.get(nodeId) ?? currentNode; - const target = targets.get(nodeId) ?? baseNode; - return [ - { - id: nodeId, - targetX: target.x, - targetY: target.y, - velocityX: 0, - velocityY: 0, - x: start.x, - y: start.y, - }, - ]; - }); - const container = containerRef.current; - const reducedMotion = window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ?? false; - let frame = 0; - let lastFrame = performance.now(); - const startedAt = lastFrame; - - const settle = (): void => { - const resolvedPositions = new Map(); - for (const particle of particles) { - resolvedPositions.set(particle.id, {x: particle.targetX, y: particle.targetY}); - } - applyGraphPositions(runtimeRef.current, resolvedPositions, layout, size, viewRef.current, labelRefs.current); - const retainedTargets = new Map(); - for (const [nodeId, target] of targets) { - const baseNode = baseLayout.nodesById.get(nodeId); - if (baseNode && Math.hypot(target.x - baseNode.x, target.y - baseNode.y) > 0.01) { - retainedTargets.set(nodeId, target); - } - } - livePositionsRef.current = retainedTargets; - setSettledPositions(retainedTargets); - container?.removeAttribute('data-layout-animating'); - }; - - if ( - reducedMotion || - particles.every(particle => Math.hypot(particle.targetX - particle.x, particle.targetY - particle.y) < 0.01) - ) { - settle(); - return; - } - - container?.setAttribute('data-layout-animating', 'true'); - const animate = (now: number): void => { - const delta = Math.min(0.032, Math.max(0.001, (now - lastFrame) / 1000)); - lastFrame = now; - let movement = 0; - const positions = new Map(); - for (const particle of particles) { - const accelerationX = (particle.targetX - particle.x) * 108 - particle.velocityX * 13; - const accelerationY = (particle.targetY - particle.y) * 108 - particle.velocityY * 13; - particle.velocityX += accelerationX * delta; - particle.velocityY += accelerationY * delta; - particle.x += particle.velocityX * delta; - particle.y += particle.velocityY * delta; - movement = Math.max( - movement, - Math.hypot(particle.targetX - particle.x, particle.targetY - particle.y), - Math.hypot(particle.velocityX, particle.velocityY) * 0.035, - ); - positions.set(particle.id, {x: particle.x, y: particle.y}); - } - livePositionsRef.current = positions; - applyGraphPositions(runtimeRef.current, positions, layout, size, viewRef.current, labelRefs.current); - if (movement < 0.08 || now - startedAt >= 1250) { - settle(); - return; - } - frame = window.requestAnimationFrame(animate); - }; - frame = window.requestAnimationFrame(animate); - return () => { - window.cancelAnimationFrame(frame); - container?.removeAttribute('data-layout-animating'); - }; - }, [animatedNeighborhoodEdges, baseLayout, focusLayoutRevision, props.selectedNodeId]); - - const labels = useMemo( - () => - visibleLabels( - layout, - props.graph.mode, - size, - view, - props.selectedNodeId, - activeNodeIds, - highlightedNodeIds, - livePositionsRef.current, - ), - [activeNodeIds, highlightedNodeIds, layout, props.graph.mode, props.selectedNodeId, size, view], - ); - - const zoomAt = (factor: number, clientX = size.width / 2, clientY = size.height / 2): void => { - setView(current => zoomViewAt(current, factor, clientX, clientY, size)); - }; - - return ( -
- { - const node = nearestNode( - layout, - view, - size, - event.nativeEvent.offsetX, - event.nativeEvent.offsetY, - livePositionsRef.current, - ); - if (node?.type === 'project') props.onOpenProject(node.projectId); - }} - onPointerDown={event => { - event.currentTarget.setPointerCapture(event.pointerId); - dragRef.current = {moved: false, pointerId: event.pointerId, x: event.clientX, y: event.clientY}; - }} - onPointerMove={event => { - const drag = dragRef.current; - if (!drag || drag.pointerId !== event.pointerId) return; - const dx = event.clientX - drag.x; - const dy = event.clientY - drag.y; - if (Math.abs(dx) + Math.abs(dy) > 2) drag.moved = true; - drag.x = event.clientX; - drag.y = event.clientY; - setView(current => ({...current, x: current.x - dx / current.zoom, y: current.y + dy / current.zoom})); - }} - onPointerUp={event => { - const drag = dragRef.current; - if (drag && !drag.moved) { - const node = nearestNode( - layout, - view, - size, - event.nativeEvent.offsetX, - event.nativeEvent.offsetY, - livePositionsRef.current, - ); - props.onSelectNode(node?.id); - } - dragRef.current = undefined; - event.currentTarget.releasePointerCapture(event.pointerId); - }} - onWheel={event => { - event.preventDefault(); - zoomAt(graphWheelZoomFactor(event.deltaY), event.nativeEvent.offsetX, event.nativeEvent.offsetY); - }} - ref={canvasRef} - /> - - {renderError ? ( -
-

GPU rendering unavailable

-

{renderError}

-
- ) : null} -
- - - -
-
- {Math.round(view.zoom * 100)}% - {view.zoom < 1.45 ? 'Zoom in to reveal symbols' : 'Detailed labels visible'} -
-
- ); -} - -function GraphSummary(props: { - readonly analysis?: GraphAnalysis; - readonly analysisError: string; - readonly analysisLoading: boolean; - readonly graph: GraphVisualization; - readonly onAnalyze: () => void; - readonly sizeMetric: GraphSizeMetric; -}): React.ReactElement { - return ( -
-

{props.graph.mode === 'overview' ? 'Repository overview' : 'Component working set'}

-

{props.graph.scope.label}

-

- {props.graph.mode === 'overview' - ? props.graph.repository.metrics === 'complete' - ? 'Node size reflects indexed symbol volume. Double-click a component to explore its symbol graph.' - : 'Node size reflects visible relationship degree. Double-click a component to explore its symbol graph.' - : `Node size reflects ${sizeMetricLabel(props.sizeMetric).toLowerCase()} among the filtered relationships.`} -

-
-
-
Indexed symbols
-
{compactNumber(props.graph.stats.totalNodes)}
-
-
-
Visible nodes
-
{compactNumber(props.graph.stats.renderedNodes)}
-
-
-
Visible links
-
{compactNumber(props.graph.stats.renderedEdges)}
-
-
-
Snapshot
-
- {props.graph.repository.snapshot.commit.slice(0, 8)} - {props.graph.repository.snapshot.dirty ? ' + dirty' : ''} -
-
- {props.graph.mode === 'overview' ? ( -
-
Overview coverage
-
- {props.graph.repository.metrics === 'deferred' - ? 'Computed on demand' - : `${compactNumber(props.graph.repository.accounting.attributedSymbols)} / ${compactNumber( - props.graph.repository.accounting.totalSymbols, - )}`} -
-
- ) : null} -
-
- - Component or facet - - - Selected node - - - Size ·{' '} - {props.graph.mode === 'overview' - ? props.graph.repository.metrics === 'complete' - ? 'Component symbols' - : 'Visible relationships' - : sizeMetricLabel(props.sizeMetric)} - -
-
-
-
-

Whole-graph analysis

-

Architecture signals

-
- -
- {props.analysis ? ( - <> -
-
-
Communities
-
- {graphAnalysisTopologyAvailable(props.analysis) - ? compactNumber(props.analysis.statistics.communityCount) - : 'Unavailable'} -
-
-
-
Components
-
- {graphAnalysisTopologyAvailable(props.analysis) - ? compactNumber(props.analysis.statistics.connectedComponentCount) - : 'Unavailable'} -
-
-
-
Hubs
-
- {graphAnalysisTopologyAvailable(props.analysis) - ? compactNumber(props.analysis.hubs.length) - : 'Unavailable'} -
-
-
-
Coverage
-
{graphAnalysisCoverageLabel(props.analysis)}
-
-
- {props.analysis.hubs.length > 0 ? ( -
-
Highest-connectivity nodes
- {props.analysis.hubs.slice(0, 4).map(hub => ( -
- - {hub.node.label} - {hub.node.path} - - - {hub.classification === 'god-node' ? 'God node' : 'Hub'} · {hub.degree} - -
- ))} -
- ) : graphAnalysisTopologyAvailable(props.analysis) ? null : ( -

Topology was not derived, so community, component, and hub absence is not inferred.

- )} - {props.analysis.surprisingLinks[0] ? ( -

- Cross-community signal: {props.analysis.surprisingLinks[0].source.label}{' '} - {relationLabel(props.analysis.surprisingLinks[0].relation)}{' '} - {props.analysis.surprisingLinks[0].target.label} -

- ) : null} - {props.analysis.warnings.length > 0 ?

{props.analysis.warnings[0]}

: null} - - ) : props.analysisError ? ( -

{props.analysisError}

- ) : ( -

Run deterministic communities, hub, and cross-boundary analysis on demand.

- )} -
-
- ); -} - -function NodeInspector(props: { - readonly detail?: GraphNodeDetail; - readonly detailError: string; - readonly detailLoading: boolean; - readonly graph: GraphVisualization; - readonly node: GraphNode; - readonly onOpenProject: () => void; - readonly onSelectNode: (nodeId: string) => void; -}): React.ReactElement { - const [tab, setTab] = useState<'evidence' | 'overview' | 'relationships'>('overview'); - useEffect(() => setTab('overview'), [props.node.id]); - const connected = props.graph.edges.filter( - edge => edge.sourceId === props.node.id || edge.targetId === props.node.id, - ); - const nodesById = new Map(props.graph.nodes.map(node => [node.id, node])); - const localRelated = connected - .slice() - .sort((left, right) => right.count - left.count || right.confidence - left.confidence) - .slice(0, 7) - .map(edge => { - const id = edge.sourceId === props.node.id ? edge.targetId : edge.sourceId; - return {edge, node: nodesById.get(id)}; - }) - .filter((item): item is {readonly edge: GraphEdge; readonly node: GraphNode} => item.node !== undefined); - const visibleNodeIds = new Set(props.graph.nodes.map(node => node.id)); - const detail = props.detail?.node.id === props.node.id ? props.detail : undefined; - const sourceLocation = detail - ? `${detail.node.path}:${detail.node.span.line}:${detail.node.span.column}` - : props.node.path; - const breadcrumb = detail ? sourceBreadcrumb(detail.node.projectId, detail.node.path) : []; - const relationshipCountsSampled = detail?.stats.summaryTruncated === true; - const relationshipSampleLabel = detail ? graphRelationshipSampleLabel(detail) : undefined; - return ( -
-
-
- {props.node.kind} - {props.node.exported ? exported : null} - {props.node.projectId !== props.graph.projectId && props.graph.mode === 'detail' ? ( - context - ) : null} -
-

{props.node.label}

-

{props.node.qualifiedName ?? props.node.projectId.replace(/^[^:]+:/, '')}

- {breadcrumb.length > 0 ? ( -
- {breadcrumb.map((part, index) => ( - - {index > 0 ? : null} - {part} - - ))} -
- ) : null} -
- {props.node.type === 'project' ? ( - - ) : ( -
- {( - [ - ['overview', 'Overview'], - ['relationships', 'Relations'], - ['evidence', 'Evidence'], - ] as const - ).map(([value, label]) => ( - - ))} -
- )} - - {props.detailLoading ? ( -
-
- ) : null} - {props.detailError ? ( -
- Detailed evidence unavailable: {props.detailError} -
- ) : null} - - {props.node.type === 'project' || tab === 'overview' ? ( - <> - {detail?.node.documentation ?

{detail.node.documentation}

: null} -
- {sourceLocation ? ( - <> -
Source
-
{sourceLocation}
- - ) : null} - {props.node.language ? ( - <> -
Language
-
{props.node.language}
- - ) : null} - {detail?.node.packageName ? ( - <> -
Package
-
{detail.node.packageName}
- - ) : null} -
{detail ? 'Fan-in' : 'Visible degree'}
-
- {detail - ? graphRelationshipCountLabel(detail.stats.incoming, relationshipCountsSampled) - : props.node.degree.toLocaleString()} -
- {detail ? ( - <> -
Fan-out
-
{graphRelationshipCountLabel(detail.stats.outgoing, relationshipCountsSampled)}
- - ) : null} - {props.node.symbolCount !== undefined ? ( - <> -
Symbols
-
{props.node.symbolCount.toLocaleString()}
-
Files
-
{props.node.fileCount?.toLocaleString()}
- - ) : null} -
- {detail?.stats.provenances.length ? ( -
- {detail.stats.provenances.map(item => ( - - {item.provenance}{' '} - {graphRelationshipCountLabel(item.count, relationshipCountsSampled)} - - ))} -
- ) : null} - {relationshipSampleLabel ?

{relationshipSampleLabel}

: null} - {(detail?.node.signature ?? props.node.signature) ? ( - 
{detail?.node.signature ?? props.node.signature}
- ) : null} - {props.node.type === 'project' && localRelated.length ? ( -
-

Strongest visible links

- {localRelated.map(({edge, node}) => ( - - ))} -
- ) : null} - - ) : null} - - {props.node.type === 'symbol' && tab === 'relationships' ? ( - detail ? ( -
-
- - {graphRelationshipCountLabel(detail.stats.incoming, relationshipCountsSampled)}{' '} - incoming - - - {graphRelationshipCountLabel(detail.stats.outgoing, relationshipCountsSampled)}{' '} - outgoing - -
-
- {detail.stats.relations.map(item => { - const maximum = detail.stats.relations[0]?.count ?? 1; - return ( -
- - {relationLabel(item.relation)} - - {graphRelationshipCountLabel(item.incoming, relationshipCountsSampled)} in ·{' '} - {graphRelationshipCountLabel(item.outgoing, relationshipCountsSampled)} out - - - -
- ); - })} -
- {relationshipSampleLabel ?

{relationshipSampleLabel}

: null} -
-

Direct neighborhood

- {detail.relationships.slice(0, 32).map(relationship => { - const canSelect = Boolean(relationship.related.id && visibleNodeIds.has(relationship.related.id)); - return ( - - ); - })} -
- {detail.stats.truncated ? ( -

Showing the strongest 160 relationships from this node.

- ) : null} -
- ) : ( -

Relationship details are not available.

- ) - ) : null} - - {props.node.type === 'symbol' && tab === 'evidence' ? ( - detail?.relationships.length ? ( -
- {detail.relationships.slice(0, 32).map(relationship => ( -
-
- {relationLabel(relationship.relation)} - {Math.round(relationship.confidence * 100)}% -
-

- {relationship.direction === 'incoming' ? 'From' : 'To'}{' '} - {relationship.related.qualifiedName ?? relationship.related.label} -

- - {relationship.evidencePath}:{relationship.evidenceSpan.line}:{relationship.evidenceSpan.column} - -
- {relationship.provenance} - - lines {relationship.evidenceSpan.line}–{relationship.evidenceSpan.endLine} - -
-
- ))} - {detail.stats.truncated ? ( -

- Evidence is capped at 160 relationships to keep inspection responsive. -

- ) : null} -
- ) : ( -

No relationship evidence is indexed for this node.

- ) - ) : null} -
- ); -} - -function GraphAdministration(props: { - readonly busy?: string; - readonly onAction: (action: GraphAdministrationAction) => void; - readonly onDiagnostics: (options: {readonly analyze: boolean; readonly deep: boolean}) => void; - readonly output?: string; - readonly report?: CodeGraphLocalDiagnosticsReport; -}): React.ReactElement { - const dialogs = useOptionalManagerDialogs(); - const [analyze, setAnalyze] = useState(false); - const [deep, setDeep] = useState(false); - const [forceCompact, setForceCompact] = useState(false); - const blocked = props.busy !== undefined; - const confirmAction = async (options: ManagerDialogOptions, action: GraphAdministrationAction): Promise => { - if (await dialogs.confirm(options)) props.onAction(action); - }; - const targetAction = async ( - managementAvailable: boolean, - action: GraphWorktreeAdministrationAction, - ): Promise => { - if (managementAvailable) return action; - const values = await dialogs.prompt({ - confirmLabel: 'Use worktree', - detail: `Checkout ${action.checkoutId.slice(-12)} · view ${action.worktreeId.slice(-8)}`, - fields: [ - { - description: 'Threadnote verifies this path against the indexed checkout and worktree before acting.', - id: 'cwd', - label: 'Absolute worktree path', - placeholder: '/absolute/path/to/worktree', - required: true, - }, - ], - message: 'Threadnote has no current local path for this indexed view.', - title: 'Locate the indexed worktree', - }); - return values ? {...action, cwd: values.cwd} : undefined; - }; - const dispatchTargetAction = async ( - managementAvailable: boolean, - action: GraphWorktreeAdministrationAction, - confirmation?: ManagerDialogOptions, - ): Promise => { - const targeted = await targetAction(managementAvailable, action); - if (!targeted) return; - if (confirmation && !(await dialogs.confirm(confirmation))) return; - props.onAction(targeted); - }; - return ( -
- - - Graph administration - - {props.report - ? graphAdministrationInventorySummary(props.report.summary) - : 'Load home-wide status, diagnostics, and maintenance controls'} - - - {props.busy ? {props.busy}… : null} - -
-
- - - - - - - -
- - {props.report ? ( -
- {props.report.databases.map(database => { - const view = database.views.find(candidate => candidate.managementAvailable) ?? database.views[0]; - const managementAvailable = view?.managementAvailable === true; - const repository = view?.repository.displayName ?? `Checkout ${database.checkoutId.slice(-8)}`; - const jobs = graphAdministrationJobSelection(database.builds, database.waiters); - const obsolete = props.report?.obsoleteStores.checkouts.find( - checkout => checkout.checkoutId === database.checkoutId, - ); - const target = view - ? graphAdministrationTarget(database.checkoutId, { - repository: view.repository, - worktreeId: view.viewWorktreeId, - }) - : undefined; - const health = database.health?.integrity ?? database.healthState; - return ( -
-
- - {repository} - {database.checkoutId.slice(-12)} - - {health === 'migration-pending' ? 'migrating' : health} -
-
-
-
Stored ready snapshots
-
- {database.health - ? database.health.readySnapshots.toLocaleString() - : database.healthState === 'deferred' - ? 'health inspection deferred' - : 'unavailable'} -
-
-
-
Active worktree views
-
{database.views.length.toLocaleString()}
-
-
-
Storage
-
- {database.storage.state === 'available' - ? formatGraphBytes(database.storage.totalBytes) - : 'missing'} -
-
-
-
Jobs
-
{jobs.total === 0 ? 'None' : `${jobs.total} actionable`}
-
-
-

- Snapshot and view counts can differ: views are per-worktree pointers, while ready snapshots are - stored graph versions that can be shared, retained for reuse, or protected while in use. -

-
- {database.views.map(candidate => { - const removalTarget = graphViewRemovalTarget(database.checkoutId, { - snapshot: candidate.snapshot, - worktreeId: candidate.viewWorktreeId, - }); - return ( -
- Active view {candidate.viewWorktreeId.slice(-8)} - - {candidate.snapshot.fileCount.toLocaleString()} files ·{' '} - {candidate.snapshot.symbolCount.toLocaleString()} symbols ·{' '} - {candidate.snapshot.edgeCount.toLocaleString()} edges - - - Folder: {graphLocalAssociationText(candidate.localAssociation)} ·{' '} - {candidate.localAssociation.state} - - {candidate.analysis ? ( - - {candidate.analysis.coverage.complete ? 'Complete' : 'Partial'} analysis ·{' '} - {candidate.analysis.coverage.topology.state === 'complete' || - candidate.analysis.coverage.topology.state === 'partial' ? ( - <> - {candidate.analysis.statistics.connectedComponentCount.toLocaleString()} components ·{' '} - {candidate.analysis.statistics.communityCount.toLocaleString()} communities · average - degree {candidate.analysis.statistics.averageDegree.toFixed(2)} · maximum{' '} - {candidate.analysis.statistics.maximumDegree.toLocaleString()} ·{' '} - {candidate.analysis.statistics.isolatedNodeCount.toLocaleString()} isolated - - ) : ( - <>topology {candidate.analysis.coverage.topology.state} - )} - - ) : null} - -
- ); - })} -
- {jobs.jobs.map(job => ( -

- View {job.identity.worktreeId.slice(-8)} · {job.state === 'running' ? 'active' : job.state} ·{' '} - {job.phase} - {job.subphase ? `/${job.subphase}` : ''} · {job.observation.liveness} - {job.error ? ` · ${job.error.summary}` : ''} -

- ))} - {jobs.hiddenCount > 0 ? ( -

+{jobs.hiddenCount} more active or failed jobs

- ) : null} - {database.issues.map(issue => ( -

- {issue.code}: {issue.message} -

- ))} -
- - - - - {obsolete ? ( - <> - - - - ) : null} - - -
- {!managementAvailable ? ( - - Index, reindex, and compact require a verified local worktree path. Purge actions target this - inventoried checkout directly. - - ) : null} -
- ); - })} -
- ) : ( -

Load diagnostics to enumerate every local graph database.

- )} - {props.output ? 
{props.output}
: null} -
-
- ); -} - -export function graphLocalAssociationText(association: CodeGraphLocalAssociation): string { - return association.displayPath ?? association.state.replaceAll('-', ' '); -} - -function GraphMaintenanceProgress(props: {readonly status: CodeGraphMaintenanceStatus}): React.ReactElement { - const {status} = props; - const elapsed = status.startedAt === undefined ? undefined : Math.max(0, Date.now() - Date.parse(status.startedAt)); - const lastUpdate = - status.updatedAt === undefined ? undefined : Math.max(0, Date.now() - Date.parse(status.updatedAt)); - const percentage = - status.completed !== undefined && status.total !== undefined && status.total > 0 - ? Math.max(0, Math.min(100, (status.completed / status.total) * 100)) - : undefined; - return ( -
-
-
-
- - {status.operation === 'selected-snapshot-purge' ? 'Selected snapshot purge' : 'Graph maintenance'} - - - {status.checkoutId ? `Checkout ${shortGraphIdentity(status.checkoutId)}` : 'Home-wide maintenance'} - {status.snapshotId ? ` · snapshot ${status.snapshotId}` : ''} - -
- {elapsed === undefined ? null : Elapsed {formatBuildDuration(elapsed)}} -
-

{graphMaintenanceStatusLabel(status)}

- {percentage === undefined ? null : ( -
- -
- )} -

- {status.completed === undefined || status.total === undefined - ? 'Waiting for the next maintenance phase update' - : `${status.completed.toLocaleString()} / ${status.total.toLocaleString()} safety phases`} - {lastUpdate === undefined ? '' : ` · last update ${formatBuildDuration(lastUpdate)} ago`} -

-
-
- ); -} - -function GraphBuildProgress(props: { - readonly build: GraphBuildStatus; - readonly repositories: readonly GraphRepositoryGroup[]; - readonly waiters: readonly GraphBuildStatus[]; -}): React.ReactElement { - const {build} = props; - const completed = build.counters.completed; - const total = build.counters.total; - const percentage = - completed !== undefined && total !== undefined && total > 0 - ? Math.max(0, Math.min(100, (completed / total) * 100)) - : undefined; - const elapsed = Math.max(0, Date.now() - Date.parse(build.timestamps.startedAt)); - const lastProgress = Math.max(0, Date.now() - Date.parse(build.timestamps.lastProgressAt)); - const progressSilent = build.coordination?.progressSilent === true; - const eta = progressSilent ? undefined : build.eta; - const target = graphBuildTarget(build, props.repositories); - const concurrency = graphBuildConcurrencyState(build, props.waiters, props.repositories); - const waiterCount = graphWaiterCountForBuild(build, props.waiters); - const statusLabel = - build.state === 'failed' - ? 'Indexing failed' - : build.state === 'queued' - ? 'Waiting to index' - : progressSilent - ? 'Indexing status is stale' - : 'Indexing'; - return ( -
-
-
- {target.repositoryLabel} - {target.worktreeLabel} -
- Elapsed {formatBuildDuration(elapsed)} -
-

- {statusLabel} · {build.phase}/{build.subphase ?? 'working'} · commit {build.identity.commit} -

-

- {build.state === 'running' - ? `Active target ${graphCommitLabel(build.identity.commit)}` - : build.state === 'queued' - ? `Queued target ${graphCommitLabel(build.identity.commit)}` - : build.state === 'failed' - ? `Failed target ${graphCommitLabel(build.identity.commit)}` - : `Completed target ${graphCommitLabel(build.identity.commit)}`} - {concurrency.latestTargetCommit === build.identity.commit - ? '' - : ` · latest target ${graphCommitLabel(concurrency.latestTargetCommit)} queued`} - {concurrency.queuedRequests === 0 - ? '' - : ` · ${concurrency.queuedRequests.toLocaleString()} queued request${concurrency.queuedRequests === 1 ? '' : 's'}`} -

- {concurrency.staleReady && concurrency.readySnapshotCommit !== undefined ? ( -

- Ready snapshot {graphCommitLabel(concurrency.readySnapshotCommit)} remains queryable · stale for latest target{' '} - {graphCommitLabel(concurrency.latestTargetCommit)} -

- ) : null} - {percentage === undefined ? null : ( -
- -
- )} -

- {build.phase === 'reclaiming' - ? `${(completed ?? 0).toLocaleString()} / ${(total ?? 0).toLocaleString()} snapshots · ${( - build.counters.pagesCompleted ?? 0 - ).toLocaleString()} pages · ${(build.counters.rowsDeleted ?? 0).toLocaleString()} rows reclaimed` - : completed === undefined || total === undefined - ? 'Preparing phase counters' - : `${completed.toLocaleString()} / ${total.toLocaleString()} ${build.counters.unit ?? 'items'}`} - {' · '}last progress change {formatBuildDuration(lastProgress)} ago -

- {progressSilent ? ( -

- No progress update for {formatBuildDuration(lastProgress)}. Process {build.owner.processId} still owns the - build lock, but Manager cannot determine whether its current operation is advancing. -

- ) : null} - {build.activity ? ( -

- Current reported step: {build.activity.stage} {build.activity.language} ·{' '} - {formatGraphBytes(build.activity.bytes)} · batch {build.activity.batchCompleted.toLocaleString()}/ - {build.activity.batchTotal.toLocaleString()} - {build.activity.sizeBucket === undefined ? '' : ` · ${build.activity.sizeBucket} source bucket`} - {build.activity.role === undefined ? '' : ` · ${build.activity.role}`} - {build.activity.classifier === undefined ? '' : `/${build.activity.classifier}`} - {build.activity.factsBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.activity.factsBytes)} emitted facts`} - {build.activity.symbols === undefined ? '' : ` · ${build.activity.symbols.toLocaleString()} symbols`} - {build.activity.relations === undefined ? '' : ` · ${build.activity.relations.toLocaleString()} relations`} - {build.activity.parseMilliseconds === undefined - ? '' - : ` · parse ${formatGraphMilliseconds(build.activity.parseMilliseconds)}`} - {build.activity.persistMilliseconds === undefined - ? '' - : ` · persist ${formatGraphMilliseconds(build.activity.persistMilliseconds)}`} - {build.activity.degraded ? ' · metadata fallback; retry scheduled' : ''} -

- ) : null} - {build.extraction ? ( -

- Extraction telemetry: {build.extraction.completedFiles.toLocaleString()} files completed ·{' '} - {build.extraction.metrics === undefined - ? '' - : `${formatGraphBytes(build.extraction.metrics.sourceBytesCompleted)}/${formatGraphBytes( - build.extraction.metrics.sourceBytesTotal, - )} source · ${formatGraphBytes(build.extraction.metrics.factsBytesCompleted)} emitted facts · ${formatGraphPercentage( - build.extraction.metrics.workUnitsCompleted, - build.extraction.metrics.workUnitsTotal, - )} class-weighted work · `} - {build.extraction.slowFiles.toLocaleString()} at or above{' '} - {formatGraphMilliseconds(CODE_GRAPH_SLOW_FILE_THRESHOLD_MILLISECONDS)} · bounded top-slow evidence{' '} - {build.extraction.topSlowFiles.length.toLocaleString()}/{CODE_GRAPH_TOP_SLOW_FILE_LIMIT.toLocaleString()} -

- ) : null} - {build.materialization?.metrics?.mode === 'full' ? ( -

- Full materialization selected - {build.materialization.metrics.fallbackReason === undefined - ? '' - : ` · incremental fallback: ${build.materialization.metrics.fallbackReason.replaceAll('-', ' ')}`} -

- ) : null} - {build.materialization?.activity ? ( -

- Current reported step: {graphMaterializationStageLabel(build.materialization.activity.stage)} · batch{' '} - {graphActiveBatchNumber( - build.materialization.activity.batchCompleted, - build.materialization.activity.batchTotal, - ).toLocaleString()} - /{build.materialization.activity.batchTotal.toLocaleString()} ·{' '} - {formatGraphBytes(build.materialization.activity.sourceBytes)} source - {build.materialization.activity.cachedFactBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.activity.cachedFactBytes)} cached facts`} - {build.materialization.activity.factsBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.activity.factsBytes)} final facts`} - {graphMaterializationRows(build.materialization.activity.rows)} - {' · '}this step{' '} - {formatBuildDuration(Math.max(0, Date.now() - Date.parse(build.materialization.activity.startedAt)))} - {build.materialization.activity.transactionMilliseconds === undefined - ? '' - : ` · transaction ${formatGraphMilliseconds(build.materialization.activity.transactionMilliseconds)}`} -

- ) : null} - {build.activation?.activity ? ( -

- Current reported step: activating · {build.activation.activity.stage.replaceAll('-', ' ')} ·{' '} - {build.activation.activity.state} - {build.activation.activity.rows === undefined - ? '' - : ` · ${build.activation.activity.rows.toLocaleString()} rows`} - {' · '}stage {formatGraphMilliseconds(build.activation.activity.stageElapsedMilliseconds)} · total{' '} - {formatGraphMilliseconds(build.activation.activity.elapsedMilliseconds)} - {build.activation.activity.transactionMilliseconds === undefined - ? '' - : ` · transaction ${formatGraphMilliseconds(build.activation.activity.transactionMilliseconds)}`} -

- ) : null} - {build.resolution?.activity ? ( -

- Reference resolution: pass {build.resolution.activity.pass.toLocaleString()} · page{' '} - {build.resolution.activity.pageCompleted.toLocaleString()}/ - {build.resolution.activity.pageTotal.toLocaleString()} ·{' '} - {build.resolution.activity.referencesCompleted.toLocaleString()}/ - {build.resolution.activity.referencesTotal.toLocaleString()} references ·{' '} - {build.resolution.activity.referencesExamined.toLocaleString()} cumulative examined ·{' '} - {build.resolution.activity.resolved.toLocaleString()} linked ·{' '} - {build.resolution.activity.aliasesDiscovered.toLocaleString()} aliases · match{' '} - {formatGraphMilliseconds(build.resolution.activity.matchingMilliseconds)} · transactions{' '} - {formatGraphMilliseconds(build.resolution.activity.transactionMilliseconds)} · total{' '} - {formatGraphMilliseconds(build.resolution.activity.elapsedMilliseconds)} -

- ) : null} - {build.materialization?.metrics ? ( - <> -

- Materialized: {build.materialization.metrics.batchesCompleted.toLocaleString()}/ - {build.materialization.metrics.batchesTotal.toLocaleString()} batches ·{' '} - {formatGraphBytes(build.materialization.metrics.sourceBytesCompleted)}/ - {formatGraphBytes(build.materialization.metrics.sourceBytesTotal)} source - {build.materialization.metrics.cachedFactBytesCompleted === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.cachedFactBytesCompleted)}${ - build.materialization.metrics.cachedFactBytesTotal === undefined - ? '' - : `/${formatGraphBytes(build.materialization.metrics.cachedFactBytesTotal)}` - } cached facts`} - {build.materialization.metrics.factsBytesCompleted === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.factsBytesCompleted)}${ - build.materialization.metrics.factsBytesTotal === undefined - ? '' - : `/${formatGraphBytes(build.materialization.metrics.factsBytesTotal)}` - } final facts`} - {graphMaterializationRows(build.materialization.metrics.rows)} - {build.materialization.metrics.loadingMilliseconds === undefined - ? '' - : ` · load ${formatGraphMilliseconds(build.materialization.metrics.loadingMilliseconds)}`} - {build.materialization.metrics.attributionMilliseconds === undefined - ? '' - : ` · attribute ${formatGraphMilliseconds(build.materialization.metrics.attributionMilliseconds)}`} - {build.materialization.metrics.transactionMilliseconds === undefined - ? '' - : ` · transactions ${formatGraphMilliseconds(build.materialization.metrics.transactionMilliseconds)}`} -

- {build.materialization.metrics.storage ? ( - <> -

- Storage: - {build.materialization.metrics.storage.durableDatabaseBytes === undefined - ? '' - : ` ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseBytes)} allocated durable pages`} - {build.materialization.metrics.storage.durableDatabaseHighWaterBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseHighWaterBytes)} allocated-page high-water`} - {build.materialization.metrics.storage.durableDatabaseGrowthHighWaterBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseGrowthHighWaterBytes)} main-database growth`} - {build.materialization.metrics.storage.durableFilesystemHighWaterBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableFilesystemHighWaterBytes)} DB + sidecars high-water`} - {build.materialization.metrics.storage.durableWalHighWaterBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableWalHighWaterBytes)} WAL high-water`} - {build.materialization.metrics.storage.durableJournalHighWaterBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableJournalHighWaterBytes)} rollback-journal high-water`} - {build.materialization.metrics.storage.durableDatabaseBytes === undefined ? '' : ' ·'}{' '} - {formatGraphBytes(build.materialization.metrics.storage.temporaryDatabaseBytes)} current TEMP database ·{' '} - {formatGraphBytes(build.materialization.metrics.storage.temporaryDatabaseHighWaterBytes)} TEMP database - high-water - {build.materialization.metrics.storage.estimatedRequiredBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedRequiredBytes)} combined estimate`} - {build.materialization.metrics.storage.estimatedTemporaryFilesystemRequiredBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedTemporaryFilesystemRequiredBytes)} TEMP-filesystem requirement`} - {build.materialization.metrics.storage.estimatedDurableFilesystemRequiredBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedDurableFilesystemRequiredBytes)} graph-filesystem requirement`} - {build.materialization.metrics.storage.temporaryAvailableBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.temporaryAvailableBytes)} available for TEMP`} - {build.materialization.metrics.storage.durableAvailableBytes === undefined - ? '' - : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableAvailableBytes)} available for graph database`} - {build.materialization.metrics.storage.filesystemsShared === true ? ' · shared filesystem' : ''} - {build.materialization.metrics.storage.materializationMode === undefined - ? '' - : ` · ${build.materialization.metrics.storage.materializationMode.replaceAll('-', ' ')}`} - {build.materialization.metrics.storage.estimateBasis === undefined - ? '' - : ` · estimate from ${build.materialization.metrics.storage.estimateBasis.replaceAll('-', ' ')}`} - {' · '}rollback journals excluded from TEMP totals -

- {graphMaterializationDiskWarning(build.materialization.metrics.storage) ? ( -

- {graphMaterializationDiskWarning(build.materialization.metrics.storage)} Indexing continues with live - storage telemetry. -

- ) : null} - - ) : null} - - ) : null} - {build.timings ? ( -

- Phase: read {formatGraphMilliseconds(build.timings.readingMilliseconds)} · parse{' '} - {formatGraphMilliseconds(build.timings.extractionMilliseconds)} · persist{' '} - {formatGraphMilliseconds(build.timings.persistenceMilliseconds)} -

- ) : null} -
- - Process {build.owner.processId} - {build.owner.processStartIdentity - ? ` · owner instance ${shortGraphIdentity(build.owner.processStartIdentity)}` - : ''} - - {eta && eta.confidence !== 'low' ? ( - - Estimated time remaining in this phase: {formatBuildDuration(eta.remainingMilliseconds)} · {eta.confidence}{' '} - confidence - {eta.basis ? ` · ${graphEtaBasisLabel(eta.basis)}` : ''} - - ) : null} - {waiterCount > 0 ? {waiterCount} waiting process(es) for this exact target : null} - {build.error ? {build.error.summary} : null} -
-
- ); -} - -function graphCommitLabel(commit: string): string { - return commit.slice(0, 12) || 'unknown'; -} - -function graphActiveBatchNumber(completed: number, total: number): number { - return total === 0 ? 0 : Math.min(total, completed + 1); -} - -function graphMaterializationStageLabel(stage: GraphMaterializationStage): string { - switch (stage) { - case 'loading-cache': - return 'loading cached facts'; - case 'attributing': - return 'attributing facts'; - case 'preparing-rows': - return 'preparing rows'; - case 'writing-analysis': - return 'writing analysis summary'; - case 'writing-symbols': - return 'writing symbols'; - case 'writing-lookups': - return 'writing lookup keys'; - case 'writing-terms': - return 'writing lexical terms'; - case 'writing-edges': - return 'writing relationships'; - case 'writing-references': - return 'writing references'; - case 'writing-receipt': - return 'recording resumable batch'; - case 'writing-candidates': - return 'writing reference candidates'; - case 'writing-facts': - return 'writing graph facts'; - case 'committing': - return 'committing batch'; - } -} - -function graphMaterializationRows(rows: GraphMaterializationRows | undefined): string { - if (!rows) return ''; - const values = [ - rows.symbols === undefined ? undefined : `${rows.symbols.toLocaleString()} symbols`, - rows.lookupKeys === undefined ? undefined : `${rows.lookupKeys.toLocaleString()} lookup keys`, - rows.terms === undefined ? undefined : `${rows.terms.toLocaleString()} terms`, - rows.edges === undefined ? undefined : `${rows.edges.toLocaleString()} relationships`, - rows.references === undefined ? undefined : `${rows.references.toLocaleString()} references`, - rows.referenceCandidates === undefined ? undefined : `${rows.referenceCandidates.toLocaleString()} candidates`, - rows.reexports === undefined ? undefined : `${rows.reexports.toLocaleString()} re-exports`, - rows.deduplicatedEdges === undefined || rows.deduplicatedEdges === 0 - ? undefined - : `${rows.deduplicatedEdges.toLocaleString()} repeated relationships collapsed`, - rows.deduplicatedReferences === undefined || rows.deduplicatedReferences === 0 - ? undefined - : `${rows.deduplicatedReferences.toLocaleString()} repeated resolution records collapsed`, - ].filter((value): value is string => value !== undefined); - return values.length > 0 ? ` · ${values.join(', ')}` : ''; -} - -function graphMaterializationDiskWarning(storage: GraphMaterializationStorage): string | undefined { - if ( - storage.filesystemsShared === true && - storage.availableBytes !== undefined && - storage.estimatedRequiredBytes !== undefined && - storage.availableBytes < storage.estimatedRequiredBytes - ) { - return 'Low disk: shared TEMP and graph storage is below the conservative combined estimate.'; - } - const scopes: string[] = []; - if ( - storage.temporaryAvailableBytes !== undefined && - storage.estimatedTemporaryFilesystemRequiredBytes !== undefined && - storage.temporaryAvailableBytes < storage.estimatedTemporaryFilesystemRequiredBytes - ) { - scopes.push('SQLite TEMP'); - } - if ( - storage.durableAvailableBytes !== undefined && - storage.estimatedDurableFilesystemRequiredBytes !== undefined && - storage.durableAvailableBytes < storage.estimatedDurableFilesystemRequiredBytes - ) { - scopes.push('graph database'); - } - return scopes.length === 0 ? undefined : `Low disk: ${scopes.join(' and ')} storage is below its estimate.`; -} - -function graphEtaBasisLabel( - basis: 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes', -): string { - switch (basis) { - case 'cached-fact-bytes': - return 'cached-fact bytes'; - case 'final-fact-bytes': - return 'final attributed fact bytes'; - case 'source-bytes': - return 'source bytes'; - case 'extraction-work': - return 'class-weighted extraction work'; - case 'files': - return 'files'; - } -} - -function formatGraphPercentage(completed: number, total: number): string { - if (total <= 0) return '0%'; - return `${Math.min(100, Math.max(0, (completed / total) * 100)).toFixed(1)}%`; -} - -function GraphEmptyState(props: {readonly building: boolean}): React.ReactElement { - return ( -
-
- ); -} - -function formatBuildDuration(milliseconds: number): string { - if (!Number.isFinite(milliseconds)) return 'unknown'; - const seconds = Math.max(0, Math.floor(milliseconds / 1_000)); - if (seconds < 60) return `${seconds}s`; - const minutes = Math.floor(seconds / 60); - if (minutes < 60) return `${minutes}m ${seconds % 60}s`; - return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; -} - -function formatGraphMilliseconds(milliseconds: number): string { - if (!Number.isFinite(milliseconds) || milliseconds < 0) return 'unknown'; - if (milliseconds < 1) return '<1ms'; - if (milliseconds < 1_000) return `${Math.round(milliseconds)}ms`; - return `${(milliseconds / 1_000).toFixed(milliseconds >= 10_000 ? 1 : 2)}s`; -} - -function formatGraphBytes(bytes: number): string { - if (!Number.isFinite(bytes) || bytes < 0) return 'unknown'; - if (bytes < 1_024) return `${Math.round(bytes)} B`; - const units = ['KiB', 'MiB', 'GiB', 'TiB']; - let value = bytes / 1_024; - let unit = units[0]!; - for (const candidate of units.slice(1)) { - if (value < 1_024) break; - value /= 1_024; - unit = candidate; - } - return `${value >= 10 ? value.toFixed(1) : value.toFixed(2)} ${unit}`; -} - -function buildGraphLayout( - graph: GraphVisualization, - sizeMetric: GraphSizeMetric, - sizingEdges: readonly GraphEdge[], -): GraphLayout { - const sizeValues = graphNodeSizeValues(sizingEdges, sizeMetric); - const nodes = graph.mode === 'overview' ? overviewLayout(graph.nodes) : detailLayout(graph.nodes, sizeValues); - const nodesById = new Map(nodes.map(node => [node.id, node])); - const extentX = Math.max(260, ...nodes.map(node => Math.abs(node.x) + node.radius)); - const extentY = Math.max(200, ...nodes.map(node => Math.abs(node.y) + node.radius)); - return {bounds: {height: extentY * 2.2, width: extentX * 2.2}, nodes, nodesById}; -} - -function graphLayoutWithPositions(layout: GraphLayout, positions: ReadonlyMap): GraphLayout { - if (positions.size === 0) return layout; - const nodes = layout.nodes.map(node => { - const position = positions.get(node.id); - return position ? {...node, x: position.x, y: position.y} : node; - }); - const nodesById = new Map(nodes.map(node => [node.id, node])); - const extentX = Math.max(260, ...nodes.map(node => Math.abs(node.x) + node.radius)); - const extentY = Math.max(200, ...nodes.map(node => Math.abs(node.y) + node.radius)); - return {bounds: {height: extentY * 2.2, width: extentX * 2.2}, nodes, nodesById}; -} - -export function graphFocusLayoutTargets( - nodes: readonly { - readonly id: string; - readonly label: string; - readonly radius: number; - readonly x: number; - readonly y: number; - }[], - selectedNodeId: string | undefined, - edges: readonly Pick[], - labelSizes: ReadonlyMap = new Map(), - zoom = FOCUS_LAYOUT_ZOOM, -): ReadonlyMap { - if (!selectedNodeId) return new Map(); - const nodesById = new Map(nodes.map(node => [node.id, node])); - const selectedNode = nodesById.get(selectedNodeId); - if (!selectedNode) return new Map(); - const neighborIds = new Set(); - for (const edge of edges) { - if (edge.sourceId === selectedNodeId && nodesById.has(edge.targetId)) neighborIds.add(edge.targetId); - if (edge.targetId === selectedNodeId && nodesById.has(edge.sourceId)) neighborIds.add(edge.sourceId); - } - neighborIds.delete(selectedNodeId); - const orderedNeighbors = [...neighborIds] - .map(nodeId => nodesById.get(nodeId)) - .filter(node => node !== undefined) - .sort((left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id)); - const highlightedIds = new Set([selectedNodeId, ...neighborIds]); - const visibleObstacles = nodes - .filter(node => !highlightedIds.has(node.id) && labelSizes.has(node.id)) - .sort((left, right) => compareCodeUnits(left.id, right.id)) - .slice(0, 180); - const focusNodes = [ - { - anchorX: selectedNode.x, - anchorY: selectedNode.y, - fixed: true, - highlighted: true, - ...selectedNode, - }, - ...orderedNeighbors.map(node => ({ - anchorX: node.x, - anchorY: node.y, - fixed: false, - highlighted: true, - ...node, - })), - ...visibleObstacles.map(node => ({ - anchorX: node.x, - anchorY: node.y, - fixed: false, - highlighted: false, - ...node, - })), - ]; - const safeZoom = Math.max(0.5, zoom); - const animatedNeighbors = focusNodes.filter(node => node.highlighted && !node.fixed); - const maximumLabelWidth = Math.max( - 72, - ...animatedNeighbors.map(node => labelSizes.get(node.id)?.width ?? Math.min(150, node.label.length * 6.2)), - ); - const maximumLabelHeight = Math.max(14, ...animatedNeighbors.map(node => labelSizes.get(node.id)?.height ?? 14)); - const columns = Math.max(2, Math.ceil(Math.sqrt((animatedNeighbors.length + 1) * 0.35))); - const rows = Math.ceil((animatedNeighbors.length + 1) / columns); - const cellWidth = (Math.min(150, maximumLabelWidth) + 14) / safeZoom; - const cellHeight = (maximumLabelHeight + 10) / safeZoom; - const slots = Array.from({length: rows * columns}, (_, index) => { - const column = index % columns; - const row = Math.floor(index / columns); - return { - x: (column - (columns - 1) / 2) * cellWidth, - y: ((rows - 1) / 2 - row) * cellHeight, - }; - }); - const centerSlot = slots.reduce( - (closest, slot, index) => - Math.hypot(slot.x, slot.y) < closest.distance ? {distance: Math.hypot(slot.x, slot.y), index} : closest, - {distance: Number.POSITIVE_INFINITY, index: 0}, - ); - slots.splice(centerSlot.index, 1); - slots.sort( - (left, right) => Math.hypot(left.x, left.y) - Math.hypot(right.x, right.y) || left.y - right.y || left.x - right.x, - ); - for (const [index, node] of animatedNeighbors.entries()) { - const slot = slots[index] ?? {x: 0, y: 0}; - node.x = selectedNode.x + slot.x; - node.y = selectedNode.y + slot.y; - node.anchorX = node.x; - node.anchorY = node.y; - let deltaX = slot.x; - let deltaY = slot.y; - let distance = Math.hypot(deltaX, deltaY); - const minimumDistance = (selectedNode.radius * 1.25 + node.radius * 1.25 + 22) / safeZoom; - if (distance < 0.001) { - const angle = (Math.abs(hashString(node.id)) % 6283) / 1000 + index * 2.399963; - deltaX = Math.cos(angle); - deltaY = Math.sin(angle); - distance = 1; - } - if (distance < minimumDistance) { - node.x = selectedNode.x + (deltaX / distance) * minimumDistance; - node.y = selectedNode.y + (deltaY / distance) * minimumDistance; - } - } - - // Preserve the full relaxation pass for ordinary neighborhoods while bounding - // maximum-cardinality focus work. Dense graphs benefit more from responsive - // interaction than from repeatedly refining already-overlapping offscreen labels. - const collisionIterations = Math.max(10, Math.min(18, Math.floor(5_000 / focusNodes.length))); - const movableFocusNodes = focusNodes.filter(node => !node.fixed); - for (let iteration = 0; iteration < collisionIterations; iteration += 1) { - for (const node of movableFocusNodes) { - node.x += (node.anchorX - node.x) * 0.006; - node.y += (node.anchorY - node.y) * 0.006; - } - for (const [leftIndex, rightIndex] of focusCollisionPairs(focusNodes, labelSizes, safeZoom)) { - separateFocusNodes(focusNodes[leftIndex]!, focusNodes[rightIndex]!, labelSizes, safeZoom); - } - for (const node of animatedNeighbors) { - const deltaX = node.x - selectedNode.x; - const deltaY = node.y - selectedNode.y; - const distance = Math.max(0.001, Math.hypot(deltaX, deltaY)); - const minimumDistance = (selectedNode.radius * 1.25 + node.radius * 1.25 + 22) / safeZoom; - if (distance < minimumDistance) { - node.x = selectedNode.x + (deltaX / distance) * minimumDistance; - node.y = selectedNode.y + (deltaY / distance) * minimumDistance; - } - } - } - return new Map(focusNodes.map(node => [node.id, {x: node.x, y: node.y}])); -} - -function focusCollisionPairs( - nodes: readonly { - readonly fixed: boolean; - readonly highlighted: boolean; - readonly id: string; - readonly label: string; - readonly radius: number; - readonly x: number; - readonly y: number; - }[], - labelSizes: ReadonlyMap, - zoom: number, -): readonly (readonly [number, number])[] { - const bounds = nodes - .map((node, index) => { - const boxes = focusNodeBoxes(node, labelSizes.get(node.id), zoom, node.fixed); - return { - bottom: Math.max(...boxes.map(box => box.bottom)), - highlighted: node.highlighted, - index, - left: Math.min(...boxes.map(box => box.left)), - right: Math.max(...boxes.map(box => box.right)), - top: Math.min(...boxes.map(box => box.top)), - }; - }) - .sort((left, right) => left.left - right.left || left.index - right.index); - const pairs: Array = []; - for (const [leftPosition, left] of bounds.entries()) { - for (let rightPosition = leftPosition + 1; rightPosition < bounds.length; rightPosition += 1) { - const right = bounds[rightPosition]!; - if (right.left >= left.right) break; - if (!left.highlighted && !right.highlighted) continue; - if (Math.min(left.bottom, right.bottom) <= Math.max(left.top, right.top)) continue; - pairs.push([left.index, right.index]); - } - } - return pairs; -} - -function separateFocusNodes( - left: { - readonly fixed: boolean; - readonly id: string; - readonly label: string; - readonly radius: number; - x: number; - y: number; - }, - right: { - readonly fixed: boolean; - readonly id: string; - readonly label: string; - readonly radius: number; - x: number; - y: number; - }, - labelSizes: ReadonlyMap, - zoom: number, -): void { - const leftBoxes = focusNodeBoxes(left, labelSizes.get(left.id), zoom, left.fixed); - const rightBoxes = focusNodeBoxes(right, labelSizes.get(right.id), zoom, right.fixed); - for (const leftBox of leftBoxes) { - for (const rightBox of rightBoxes) { - const overlapX = Math.min(leftBox.right, rightBox.right) - Math.max(leftBox.left, rightBox.left); - const overlapY = Math.min(leftBox.bottom, rightBox.bottom) - Math.max(leftBox.top, rightBox.top); - if (overlapX <= 0 || overlapY <= 0) continue; - const leftCenterX = (leftBox.left + leftBox.right) / 2; - const leftCenterY = (leftBox.top + leftBox.bottom) / 2; - const rightCenterX = (rightBox.left + rightBox.right) / 2; - const rightCenterY = (rightBox.top + rightBox.bottom) / 2; - const fallback = hashString(`${left.id}:${right.id}`); - if (overlapX < overlapY) { - const direction = - leftCenterX === rightCenterX ? (fallback % 2 === 0 ? -1 : 1) : Math.sign(leftCenterX - rightCenterX); - moveFocusPair(left, right, direction * (overlapX + 2 / zoom), 0); - } else { - const direction = - leftCenterY === rightCenterY ? (fallback % 2 === 0 ? -1 : 1) : Math.sign(leftCenterY - rightCenterY); - moveFocusPair(left, right, 0, direction * (overlapY + 2 / zoom)); - } - } - } -} - -function moveFocusPair( - left: {readonly fixed: boolean; x: number; y: number}, - right: {readonly fixed: boolean; x: number; y: number}, - deltaX: number, - deltaY: number, -): void { - if (left.fixed && right.fixed) return; - if (left.fixed) { - right.x -= deltaX; - right.y -= deltaY; - return; - } - if (right.fixed) { - left.x += deltaX; - left.y += deltaY; - return; - } - left.x += deltaX / 2; - left.y += deltaY / 2; - right.x -= deltaX / 2; - right.y -= deltaY / 2; -} - -function focusNodeBoxes( - node: {readonly label: string; readonly radius: number; readonly x: number; readonly y: number}, - measured: {readonly height: number; readonly width: number} | undefined, - zoom: number, - selected: boolean, -): readonly {readonly bottom: number; readonly left: number; readonly right: number; readonly top: number}[] { - const nodeHalfSize = (node.radius * 1.25 + 4) / zoom; - const estimatedWidth = Math.min(selected ? 300 : 220, Math.max(28, node.label.length * 6.2 + (selected ? 14 : 0))); - const labelWidth = (measured?.width ?? estimatedWidth) / zoom; - const labelHeight = (measured?.height ?? (selected ? 22 : 14)) / zoom; - const labelLeft = node.x + (node.radius + 4) / zoom; - const margin = 3 / zoom; - const nodeBox = { - bottom: node.y + nodeHalfSize + margin, - left: node.x - nodeHalfSize - margin, - right: node.x + nodeHalfSize + margin, - top: node.y - nodeHalfSize - margin, - }; - if (!measured && !selected) return [nodeBox]; - return [ - nodeBox, - { - bottom: node.y + labelHeight / 2 + margin, - left: labelLeft - margin, - right: labelLeft + labelWidth + margin, - top: node.y - labelHeight / 2 - margin, - }, - ]; -} - -function overviewLayout(nodes: readonly GraphNode[]): readonly PositionedNode[] { - const ordered = [...nodes].sort( - (left, right) => - (right.symbolCount ?? right.degree) - (left.symbolCount ?? left.degree) || - compareCodeUnits(left.label, right.label), - ); - return ordered.map((node, index) => { - const angle = index * 2.399963; - const ring = index === 0 ? 0 : 78 + Math.sqrt(index) * 84; - return positionNode(node, Math.cos(angle) * ring, Math.sin(angle) * ring, index); - }); -} - -function detailLayout(nodes: readonly GraphNode[], sizeValues: ReadonlyMap): readonly PositionedNode[] { - const groups = new Map(); - for (const node of nodes) { - const group = graphGroup(node); - const items = groups.get(group) ?? []; - items.push(node); - groups.set(group, items); - } - const orderedGroups = [...groups].sort( - ([leftName, left], [rightName, right]) => right.length - left.length || compareCodeUnits(leftName, rightName), - ); - const output: PositionedNode[] = []; - for (const [groupIndex, [, items]] of orderedGroups.entries()) { - const groupAngle = groupIndex * 2.399963; - const groupRadius = orderedGroups.length === 1 ? 0 : 120 + Math.sqrt(groupIndex) * 135; - const centerX = Math.cos(groupAngle) * groupRadius; - const centerY = Math.sin(groupAngle) * groupRadius; - const ordered = [...items].sort( - (left, right) => right.degree - left.degree || compareCodeUnits(left.label, right.label), - ); - for (const [itemIndex, node] of ordered.entries()) { - const angle = itemIndex * 2.399963 + groupAngle; - const radius = itemIndex === 0 ? 0 : 17 * Math.sqrt(itemIndex); - output.push( - positionNode( - node, - centerX + Math.cos(angle) * radius, - centerY + Math.sin(angle) * radius, - groupIndex, - sizeValues.get(node.id) ?? 0, - ), - ); - } - } - return output; -} - -function positionNode( - node: GraphNode, - x: number, - y: number, - colorIndex: number, - sizeValue = node.degree, -): PositionedNode { - const radius = - node.type === 'project' - ? 8 + Math.min(14, Math.sqrt(Math.max(1, Math.log2((node.symbolCount ?? sizeValue) + 1))) * 3) - : 4 + Math.min(11, Math.log2(Math.max(0, sizeValue) + 1) * 2); - return { - ...node, - color: new THREE.Color(colorForNode(node, colorIndex)), - radius, - x, - y, - }; -} - -export function graphNodeSizeValues( - edges: readonly Pick[], - metric: GraphSizeMetric, -): ReadonlyMap { - const connected = new Map>(); - const add = (nodeId: string, neighborId: string): void => { - const neighbors = connected.get(nodeId) ?? new Set(); - neighbors.add(neighborId); - connected.set(nodeId, neighbors); - }; - for (const edge of edges) { - if (edge.sourceId === edge.targetId) continue; - if (metric !== 'incoming') add(edge.sourceId, edge.targetId); - if (metric !== 'outgoing') add(edge.targetId, edge.sourceId); - } - return new Map([...connected].map(([nodeId, neighbors]) => [nodeId, neighbors.size])); -} - -function colorForNode(node: GraphNode, fallbackIndex: number): string { - if (node.type === 'project') return GRAPH_PALETTE[fallbackIndex % GRAPH_PALETTE.length]!; - const key = node.projectId || node.kind; - return GRAPH_PALETTE[Math.abs(hashString(key)) % GRAPH_PALETTE.length]!; -} - -function graphGroup(node: GraphNode): string { - if (!node.path) return node.projectId; - const parts = node.path.split('/'); - return parts.slice(0, Math.min(2, Math.max(1, parts.length - 1))).join('/'); -} - -function fittedView(layout: GraphLayout, size: {readonly height: number; readonly width: number}): ViewState { - const padding = 1.12; - const zoom = Math.min( - 1.6, - Math.max( - MIN_ZOOM, - Math.min(size.width / (layout.bounds.width * padding), size.height / (layout.bounds.height * padding)), - ), - ); - return {x: 0, y: 0, zoom: Number.isFinite(zoom) ? zoom : 1}; -} - -export function graphFocusTarget( - current: ViewState, - node: {readonly x: number; readonly y: number}, - mode: GraphVisualization['mode'], -): ViewState { - const targetZoom = SEARCH_FOCUS_ZOOM[mode]; - const currentZoom = Number.isFinite(current.zoom) ? current.zoom : targetZoom; - return { - x: Number.isFinite(node.x) ? node.x : Number.isFinite(current.x) ? current.x : 0, - y: Number.isFinite(node.y) ? node.y : Number.isFinite(current.y) ? current.y : 0, - zoom: Math.min(targetZoom * 1.35, Math.max(currentZoom, targetZoom)), - }; -} - -export function graphWheelZoomFactor(deltaY: number): number { - if (Number.isNaN(deltaY)) return 1; - return Math.max(0.72, Math.min(1.38, Math.exp(-deltaY * 0.0012))); -} - -function updateCamera( - camera: THREE.OrthographicCamera, - view: ViewState, - size: {readonly height: number; readonly width: number}, -): void { - camera.left = -size.width / 2 / view.zoom; - camera.right = size.width / 2 / view.zoom; - camera.top = size.height / 2 / view.zoom; - camera.bottom = -size.height / 2 / view.zoom; - camera.near = 0.1; - camera.far = 200; - camera.position.set(view.x, view.y, 100); - camera.updateProjectionMatrix(); -} - -function graphPosition( - node: {readonly id: string; readonly x: number; readonly y: number}, - positions?: ReadonlyMap, -): GraphPosition { - return positions?.get(node.id) ?? node; -} - -function applyGraphPositions( - runtime: GraphRuntime | undefined, - positions: ReadonlyMap, - layout: GraphLayout, - size: {readonly height: number; readonly width: number}, - view: ViewState, - labelElements: ReadonlyMap, -): void { - if (runtime) { - for (const [index, nodeId] of runtime.nodeIds.entries()) { - const node = layout.nodesById.get(nodeId); - if (!node) continue; - const position = graphPosition(node, positions); - runtime.nodePosition.setXYZ(index, position.x, position.y, 1); - } - runtime.nodePosition.needsUpdate = true; - for (const [index, edge] of runtime.edges.entries()) { - const source = layout.nodesById.get(edge.sourceId); - const target = layout.nodesById.get(edge.targetId); - if (!source || !target) continue; - const sourcePosition = graphPosition(source, positions); - const targetPosition = graphPosition(target, positions); - runtime.edgePosition.setXYZ(index * 2, sourcePosition.x, sourcePosition.y, 0); - runtime.edgePosition.setXYZ(index * 2 + 1, targetPosition.x, targetPosition.y, 0); - } - runtime.edgePosition.needsUpdate = true; - if (runtime.highlightPosition) { - const highlightPositions = directionalEdgePositions(runtime.highlightedEdges, layout.nodesById, positions); - if (highlightPositions.length === runtime.highlightPosition.array.length) { - runtime.highlightPosition.array.set(highlightPositions); - runtime.highlightPosition.needsUpdate = true; - } - } - if (runtime.selectedNodeId && runtime.selectedPosition) { - const selectedNode = layout.nodesById.get(runtime.selectedNodeId); - if (selectedNode) { - const selectedPosition = graphPosition(selectedNode, positions); - runtime.selectedPosition.setXYZ(0, selectedPosition.x, selectedPosition.y, 2); - runtime.selectedPosition.needsUpdate = true; - } - } - } - - for (const [nodeId, element] of labelElements) { - const node = layout.nodesById.get(nodeId); - if (!node) continue; - const position = graphPosition(node, positions); - const x = size.width / 2 + (position.x - view.x) * view.zoom; - const y = size.height / 2 - (position.y - view.y) * view.zoom; - element.style.left = `${x + node.radius + 4}px`; - element.style.top = `${y}px`; - } - if (runtime) runtime.renderer.render(runtime.scene, runtime.camera); -} - -function zoomViewAt( - view: ViewState, - factor: number, - screenX: number, - screenY: number, - size: {readonly height: number; readonly width: number}, -): ViewState { - const zoom = Math.max(MIN_ZOOM, Math.min(MAX_ZOOM, view.zoom * factor)); - const dx = screenX - size.width / 2; - const dy = screenY - size.height / 2; - const worldX = view.x + dx / view.zoom; - const worldY = view.y - dy / view.zoom; - return {x: worldX - dx / zoom, y: worldY + dy / zoom, zoom}; -} - -function isAbortError(cause: unknown): boolean { - return cause instanceof DOMException - ? cause.name === 'AbortError' - : cause instanceof Error && cause.name === 'AbortError'; -} - -function nearestNode( - layout: GraphLayout, - view: ViewState, - size: {readonly height: number; readonly width: number}, - screenX: number, - screenY: number, - positions?: ReadonlyMap, -): PositionedNode | undefined { - const worldX = view.x + (screenX - size.width / 2) / view.zoom; - const worldY = view.y - (screenY - size.height / 2) / view.zoom; - let selected: PositionedNode | undefined; - let selectedDistance = Number.POSITIVE_INFINITY; - for (const node of layout.nodes) { - const position = graphPosition(node, positions); - const distance = Math.hypot(position.x - worldX, position.y - worldY); - const hitRadius = Math.max(node.radius * 1.45, 10 / view.zoom); - if (distance <= hitRadius && distance < selectedDistance) { - selected = node; - selectedDistance = distance; - } - } - return selected; -} - -function visibleLabels( - layout: GraphLayout, - mode: GraphVisualization['mode'], - size: {readonly height: number; readonly width: number}, - view: ViewState, - selectedNodeId?: string, - activeNodeIds?: ReadonlySet, - highlightedNodeIds?: ReadonlySet, - positions?: ReadonlyMap, -): readonly {readonly node: PositionedNode; readonly x: number; readonly y: number}[] { - const baseMaximum = - mode === 'overview' - ? view.zoom < 0.65 - ? 18 - : 80 - : view.zoom < 0.75 - ? 8 - : view.zoom < 1.45 - ? 24 - : view.zoom < 3 - ? 72 - : 180; - const highlightedMaximum = - view.zoom < 0.75 - ? 0 - : view.zoom < 1.45 - ? Math.min(24, highlightedNodeIds?.size ?? 0) - : Math.min(MAX_FOCUSED_LABELS + 1, highlightedNodeIds?.size ?? 0); - const maximum = Math.max(baseMaximum, highlightedMaximum); - let focusedLabelCount = 0; - return [...layout.nodes] - .filter(node => !activeNodeIds || activeNodeIds.has(node.id)) - .flatMap(node => { - const position = graphPosition(node, positions); - const x = size.width / 2 + (position.x - view.x) * view.zoom; - const y = size.height / 2 - (position.y - view.y) * view.zoom; - return x < -80 || x > size.width + 80 || y < -30 || y > size.height + 30 ? [] : [{node, x, y}]; - }) - .sort((left, right) => { - if (left.node.id === selectedNodeId) return -1; - if (right.node.id === selectedNodeId) return 1; - if (highlightedNodeIds?.has(left.node.id) && !highlightedNodeIds.has(right.node.id)) return -1; - if (highlightedNodeIds?.has(right.node.id) && !highlightedNodeIds.has(left.node.id)) return 1; - return ( - right.node.degree - left.node.degree || - right.node.radius - left.node.radius || - compareCodeUnits(left.node.label, right.node.label) - ); - }) - .filter(({node}) => { - if (node.id === selectedNodeId || !highlightedNodeIds?.has(node.id)) return true; - focusedLabelCount += 1; - return focusedLabelCount <= MAX_FOCUSED_LABELS; - }) - .map(({node, x, y}) => ({node, x: x + node.radius + 4, y})) - .slice(0, maximum); -} - -function directionalEdgePositions( - edges: readonly GraphEdge[], - nodesById: ReadonlyMap, - positionOverrides?: ReadonlyMap, -): readonly number[] { - const positions: number[] = []; - for (const edge of edges.slice(0, MAX_ANIMATED_NEIGHBOR_EDGES)) { - const source = nodesById.get(edge.sourceId); - const target = nodesById.get(edge.targetId); - if (!source || !target) continue; - const sourcePosition = graphPosition(source, positionOverrides); - const targetPosition = graphPosition(target, positionOverrides); - let dx = targetPosition.x - sourcePosition.x; - let dy = targetPosition.y - sourcePosition.y; - let length = Math.hypot(dx, dy); - if (length < 0.001) { - const angle = (Math.abs(hashString(edge.id)) % 6283) / 1000; - dx = Math.cos(angle) * 0.001; - dy = Math.sin(angle) * 0.001; - length = 0.001; - } - const unitX = dx / length; - const unitY = dy / length; - const tipX = targetPosition.x - unitX * (target.radius + 2); - const tipY = targetPosition.y - unitY * (target.radius + 2); - const arrowLength = Math.min(8, Math.max(4, length * 0.16)); - const wingX = tipX - unitX * arrowLength; - const wingY = tipY - unitY * arrowLength; - const normalX = -unitY * arrowLength * 0.55; - const normalY = unitX * arrowLength * 0.55; - positions.push( - sourcePosition.x, - sourcePosition.y, - 1.5, - tipX, - tipY, - 1.5, - tipX, - tipY, - 1.5, - wingX + normalX, - wingY + normalY, - 1.5, - tipX, - tipY, - 1.5, - wingX - normalX, - wingY - normalY, - 1.5, - ); - } - return positions; -} - -function graphPointMaterial(scale: number, zoom: number): THREE.ShaderMaterial { - return new THREE.ShaderMaterial({ - blending: THREE.AdditiveBlending, - depthWrite: false, - fragmentShader: ` - varying vec3 vColor; - void main() { - vec2 point = gl_PointCoord - vec2(0.5); - float distanceToCenter = length(point); - if (distanceToCenter > 0.5) discard; - float glow = smoothstep(0.5, 0.05, distanceToCenter); - float core = smoothstep(0.24, 0.05, distanceToCenter); - gl_FragColor = vec4(vColor + core * 0.32, glow * 0.94); - } - `, - transparent: true, - uniforms: { - viewScale: {value: graphPointViewScale(zoom)}, - }, - vertexColors: true, - vertexShader: ` - attribute float pointSize; - uniform float viewScale; - varying vec3 vColor; - void main() { - vColor = color; - gl_PointSize = max(3.0, pointSize * ${scale.toFixed(2)} * viewScale); - gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); - } - `, - }); -} - -function graphPointViewScale(zoom: number): number { - return Math.min(1.25, Math.max(0.32, zoom * 0.75)); -} - -function compactNumber(value: number): string { - return new Intl.NumberFormat(undefined, {maximumFractionDigits: 1, notation: 'compact'}).format(value); -} - -function graphProjectBadge(project: GraphProject): string { - if (project.model === 'legacy-fallback') return 'legacy group'; - if (project.model === 'facet') return 'facet'; - return project.buildSystem ? `${project.buildSystem} ${project.kind ?? 'component'}` : (project.kind ?? 'component'); -} - -function relationLabel(value: string): string { - return value.replaceAll('_', ' ').replace(/\b\w/g, letter => letter.toUpperCase()); -} - -function sizeMetricLabel(metric: GraphSizeMetric): string { - switch (metric) { - case 'incoming': - return 'Distinct incoming neighbors'; - case 'outgoing': - return 'Distinct outgoing neighbors'; - default: - return 'Distinct connections'; - } -} - -function sourceBreadcrumb(projectId: string, path: string): readonly string[] { - const project = projectId.replace(/^[^:]+:/, ''); - const parts = path.split('/').filter(Boolean); - const compactPath = parts.length > 4 ? [...parts.slice(0, 2), '…', ...parts.slice(-2)] : parts; - return [project, ...compactPath.filter((part, index) => index > 0 || part !== project)]; -} - -function hashString(value: string): number { - let hash = 2166136261; - for (let index = 0; index < value.length; index += 1) { - hash ^= value.charCodeAt(index); - hash = Math.imul(hash, 16777619); - } - return hash | 0; -} - -function lerp(start: number, end: number, progress: number): number { - return start + (end - start) * progress; -} +export { + cacheGraphNodeDetail, + createGraphQueryRequestGate, + graphAdministrationJobSelection, + graphAdministrationTarget, + graphAnalysisCoverageLabel, + graphAnalysisRequestIsCurrent, + graphAnalysisTopologyAvailable, + graphBuildConcurrencyState, + graphBuildIsActive, + graphBuildShouldDisplay, + graphBuildTarget, + graphCatalogContinuationHasMore, + graphCatalogPageOffsets, + graphCatalogSearchOptions, + graphCompletedBuildResultIdentity, + graphDiagnosticsRequiresCatalogRefresh, + graphDisplayEdges, + graphLocalAssociationText, + graphMaintenanceStatusLabel, + mergeGraphCatalogStatus, + graphNodeDetailRequestIsCurrent, + graphNodeSizeValues, + graphOverviewSizeLabel, + graphQueryRequestIsCurrent, + graphRelationshipCountLabel, + graphRelationshipSampleLabel, + graphRepositoryOptionLabel, + graphRequestIsCurrent, + graphStatusPollDelay, + graphStatusRequiresCatalogRefresh, + graphViewRemovalTarget, + graphWaiterCountForBuild, + graphWithNodeNeighborhood, + managerGraphDebouncedQueryCandidate, + managerGraphQueryCandidate, + mergeGraphRepositoryGroups, + resolveGraphSelection, + type GraphAdministrationAction, + type GraphAdministrationJobSelection, + type GraphAnalysis, + type GraphBuildConcurrencyState, + type GraphBuildStatus, + type GraphBuildTarget, + type GraphCatalog, + type GraphCatalogDiagnostic, + type GraphCatalogPage, + type GraphCatalogSearchOptions, + type GraphEdge, + type GraphFocusMode, + type GraphNodeDetail, + type GraphPosition, + type GraphQueryMetadata, + type GraphQueryRequestGate, + type GraphQueryRequestHandle, + type GraphQueryRequestInput, + type GraphQueryRequestOutcome, + type GraphQueryVisualization, + type GraphRepository, + type GraphRepositoryGroup, + type GraphSizeMetric, + type GraphStorageSummary, + type GraphViewPage, + type GraphVisualization, + type ViewState, +} from './manager_graph_model.js'; +export { + graphFocusLayoutTargets, + graphFocusTarget, + graphWheelZoomFactor, + managerGraphClientRenderProxy, +} from './manager_graph_scene.js'; +export {GraphWorkspace} from './manager_graph_workspace.js'; diff --git a/src/manager_graph_model.ts b/src/manager_graph_model.ts new file mode 100644 index 00000000..aa687947 --- /dev/null +++ b/src/manager_graph_model.ts @@ -0,0 +1,1417 @@ +import type * as THREE from 'three'; +import type {CodeGraphAutomaticCompactionStatus} from './code_graph/automatic_compaction.js'; +import type {CodeGraphLocalDiagnosticsReport} from './code_graph/diagnostics.js'; +import type {CodeGraphLocalAssociation} from './code_graph/local_provenance.js'; +import type {CodeGraphMaintenanceStatus} from './code_graph/maintenance_gate.js'; +import type {ManagerGraphStorageSummary} from './code_graph/manager_status.js'; +import {compareCodeUnits} from './code_graph/ordering.js'; +import { + MANAGER_GRAPH_DEFAULT_EDGE_LIMIT, + MANAGER_GRAPH_DEFAULT_NODE_LIMIT, + MANAGER_GRAPH_MAX_EDGE_LIMIT, + MANAGER_GRAPH_MAX_NODE_LIMIT, +} from './manager_graph_limits.js'; + +export interface GraphProject { + readonly buildSystem?: string; + readonly fileCount?: number; + readonly id: string; + readonly kind?: string; + readonly label: string; + readonly model?: 'component' | 'facet' | 'legacy-fallback'; + readonly provenance?: string; + readonly symbolCount?: number; + readonly workspaceId?: string; +} + +export interface GraphWorkspaceDescriptor { + readonly buildSystem: string; + readonly id: string; + readonly name: string; + readonly root: string; +} + +export interface GraphSnapshot { + readonly commit: string; + readonly completedAt?: string; + readonly dirty: boolean; + readonly edgeCount: number; + readonly fileCount: number; + readonly id: string; + readonly symbolCount: number; +} + +export interface GraphRepository { + readonly accounting: { + readonly attributedSymbols: number; + readonly componentSymbols: number; + readonly fallbackSymbols: number; + readonly omittedSymbols: number; + readonly totalSymbols: number; + }; + readonly activatedAt?: string; + readonly checkoutId: string; + readonly displayName: string; + readonly id: string; + readonly label: string; + readonly localAssociation: CodeGraphLocalAssociation; + readonly metrics: 'complete' | 'deferred'; + readonly model: 'legacy-fallback' | 'workspace'; + readonly projectCount: number; + readonly projects: readonly GraphProject[]; + readonly projectsTruncated: boolean; + readonly snapshot: GraphSnapshot; + readonly worktreeId: string; + readonly workspaceCount: number; + readonly workspaces: readonly GraphWorkspaceDescriptor[]; + readonly workspacesTruncated: boolean; +} + +export interface GraphRepositoryGroup { + readonly defaultViewId: string; + readonly displayName: string; + readonly id: string; + readonly repositoryId: string; + readonly views: readonly GraphRepository[]; + readonly viewsTruncated: boolean; +} + +export interface GraphCatalogDiagnostic { + readonly checkoutId: string; + readonly code: 'lease-deferred' | 'lease-failed' | 'no-ready-snapshot' | 'unreadable-database'; + readonly message: string; +} + +export interface GraphCatalog { + readonly automaticCompaction?: CodeGraphAutomaticCompactionStatus; + readonly builds: readonly GraphBuildStatus[]; + readonly catalogRevision?: string; + readonly diagnostics: readonly GraphCatalogDiagnostic[]; + readonly lifecyclePending?: boolean; + readonly maintenance?: CodeGraphMaintenanceStatus; + readonly repositories: readonly GraphRepositoryGroup[]; + readonly storage?: Readonly>; + readonly waiterCount: number; + readonly waiters: readonly GraphBuildStatus[]; +} + +export type GraphAdministrationAction = + | { + readonly action: 'compact' | 'index'; + readonly checkoutId: string; + readonly cwd?: string; + readonly dryRun?: boolean; + readonly force?: boolean; + readonly full?: boolean; + readonly repositoryId: string; + readonly worktreeId: string; + } + | { + readonly action: 'purge' | 'purge-obsolete'; + readonly checkoutId: string; + readonly dryRun?: boolean; + } + | { + readonly action: 'remove-view'; + readonly checkoutId: string; + readonly dryRun?: boolean; + readonly expectedSnapshotId: string; + readonly worktreeId: string; + } + | {readonly action: 'purge-all'; readonly dryRun?: boolean} + | {readonly action: 'repair'; readonly deep?: boolean; readonly dryRun?: boolean}; + +export type GraphWorktreeAdministrationAction = Extract< + GraphAdministrationAction, + {readonly action: 'compact' | 'index'} +>; + +export function graphAdministrationTarget( + checkoutId: string, + view: {readonly repository: {readonly repositoryId: string}; readonly worktreeId: string}, +): Pick { + return {checkoutId, repositoryId: view.repository.repositoryId, worktreeId: view.worktreeId}; +} + +export function graphViewRemovalTarget( + checkoutId: string, + view: {readonly snapshot: {readonly id: string}; readonly worktreeId: string}, +): Pick< + Extract, + 'checkoutId' | 'expectedSnapshotId' | 'worktreeId' +> { + return {checkoutId, expectedSnapshotId: view.snapshot.id, worktreeId: view.worktreeId}; +} + +export interface GraphCatalogPage { + readonly projectOffset: number; + readonly query: string; + readonly repository: GraphRepository; + readonly workspaceOffset: number; +} + +export interface GraphViewPage { + readonly hasMore: boolean; + readonly offset: number; + readonly query: string; + readonly repositories: readonly GraphRepositoryGroup[]; +} + +export interface GraphBuildStatus { + readonly activation?: { + readonly activity: { + readonly elapsedMilliseconds: number; + readonly rows?: number; + readonly stage: GraphActivationStage; + readonly stageElapsedMilliseconds: number; + readonly startedAt: string; + readonly state: 'completed' | 'progress' | 'started'; + readonly transactionMilliseconds?: number; + }; + }; + readonly activity?: { + readonly batchCompleted: number; + readonly batchTotal: number; + readonly bytes: number; + readonly classifier?: string; + readonly degraded?: boolean; + readonly factsBytes?: number; + readonly language: string; + readonly parseMilliseconds?: number; + readonly persistMilliseconds?: number; + readonly relations?: number; + readonly role?: string; + readonly sizeBucket?: '0-16KiB' | '16-64KiB' | '64-256KiB' | '256KiB-1MiB' | '>1MiB'; + readonly stage: 'extracting' | 'persisting' | 'reading'; + readonly symbols?: number; + }; + readonly buildId: string; + readonly coordination?: { + readonly lockVerified: boolean; + readonly progressSilent?: boolean; + readonly role: 'history' | 'owner' | 'waiter'; + }; + readonly counters: { + readonly accepted?: number; + readonly completed?: number; + readonly edges?: number; + readonly excluded?: number; + readonly pagesCompleted?: number; + readonly reused?: number; + readonly resolved?: number; + readonly rowsDeleted?: number; + readonly skipped?: number; + readonly symbols?: number; + readonly total?: number; + readonly unit?: string; + }; + readonly error?: {readonly summary: string}; + readonly eta?: { + readonly basis?: 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes'; + readonly confidence: 'high' | 'low' | 'medium'; + readonly remainingMilliseconds: number; + }; + readonly extraction?: { + readonly completedFiles: number; + readonly metrics?: { + readonly factsBytesCompleted: number; + readonly sourceBytesCompleted: number; + readonly sourceBytesTotal: number; + readonly workUnitsCompleted: number; + readonly workUnitsTotal: number; + }; + readonly slowFiles: number; + readonly topSlowFiles: readonly { + readonly classifier: string; + readonly degraded?: boolean; + readonly durationMilliseconds: number; + readonly extension: string; + readonly factsBytes?: number; + readonly language: string; + readonly pathHash: string; + readonly relations?: number; + readonly role: string; + readonly sizeBucket: '0-16KiB' | '16-64KiB' | '64-256KiB' | '256KiB-1MiB' | '>1MiB'; + readonly sourceBytes: number; + readonly symbols?: number; + }[]; + }; + readonly identity: { + readonly checkoutId: string; + readonly commit: string; + readonly displayName?: string; + readonly repositoryId: string; + readonly worktreeId: string; + }; + readonly managerContext?: { + readonly branch?: string; + readonly worktreePath: string; + }; + readonly observation: { + readonly heartbeatAgeMilliseconds: number; + readonly liveness: 'abandoned' | 'active' | 'completed' | 'failed' | 'stalled'; + }; + readonly materialization?: { + readonly activity?: { + readonly batchCompleted: number; + readonly batchTotal: number; + readonly cachedFactBytes?: number; + readonly elapsedMilliseconds?: number; + readonly factsBytes?: number; + readonly rows?: GraphMaterializationRows; + readonly sourceBytes: number; + readonly stage: GraphMaterializationStage; + readonly stageElapsedMilliseconds?: number; + readonly startedAt: string; + readonly transactionMilliseconds?: number; + }; + readonly metrics?: { + readonly attributionMilliseconds?: number; + readonly batchesCompleted: number; + readonly batchesTotal: number; + readonly cachedFactBytesCompleted?: number; + readonly cachedFactBytesTotal?: number; + readonly fallbackReason?: string; + readonly factsBytesCompleted?: number; + readonly factsBytesTotal?: number; + readonly loadingMilliseconds?: number; + readonly mode?: 'full' | 'incremental-clean' | 'incremental-overlay'; + readonly rows?: GraphMaterializationRows; + readonly sourceBytesCompleted: number; + readonly sourceBytesTotal: number; + readonly stageMilliseconds?: Readonly>>; + readonly storage?: GraphMaterializationStorage; + readonly transactionMilliseconds?: number; + }; + }; + readonly owner: {readonly processId: number; readonly processStartIdentity?: string}; + readonly phase: string; + readonly request?: {readonly key: string}; + readonly resolution?: { + readonly activity: { + readonly aliasesDiscovered: number; + readonly elapsedMilliseconds: number; + readonly matchingMilliseconds: number; + readonly pageCompleted: number; + readonly pageTotal: number; + readonly pagesCompleted: number; + readonly pass: number; + readonly referencesCompleted: number; + readonly referencesExamined: number; + readonly referencesTotal: number; + readonly resolved: number; + readonly startedAt: string; + readonly transactionMilliseconds: number; + }; + }; + readonly result?: {readonly snapshotId: string}; + readonly state: 'completed' | 'failed' | 'queued' | 'running'; + readonly subphase?: string; + readonly timings?: { + readonly extractionMilliseconds: number; + readonly persistenceMilliseconds: number; + readonly readingMilliseconds: number; + }; + readonly timestamps: { + readonly heartbeatAt: string; + readonly lastProgressAt: string; + readonly startedAt: string; + }; +} + +export type GraphActivationStage = + | 'checkpointing-snapshot' + | 'committing-snapshot' + | 'copying-edges' + | 'copying-files' + | 'copying-lookup-keys' + | 'copying-reexports' + | 'copying-symbols' + | 'copying-terms' + | 'copying-workspace' + | 'recording-completion' + | 'validating-input'; + +export type GraphMaterializationStage = + | 'attributing' + | 'committing' + | 'loading-cache' + | 'preparing-rows' + | 'writing-analysis' + | 'writing-candidates' + | 'writing-edges' + | 'writing-facts' + | 'writing-lookups' + | 'writing-references' + | 'writing-receipt' + | 'writing-symbols' + | 'writing-terms'; + +export interface GraphMaterializationRows { + readonly deduplicatedEdges?: number; + readonly deduplicatedReferences?: number; + readonly edges?: number; + readonly lookupKeys?: number; + readonly referenceCandidates?: number; + readonly references?: number; + readonly reexports?: number; + readonly symbols?: number; + readonly terms?: number; +} + +export interface GraphMaterializationStorage { + readonly availableBytes?: number; + readonly durableAvailableBytes?: number; + readonly durableDatabaseBytes?: number; + readonly durableDatabaseFileBytes?: number; + readonly durableDatabaseFileHighWaterBytes?: number; + readonly durableDatabaseGrowthBytes?: number; + readonly durableDatabaseGrowthHighWaterBytes?: number; + readonly durableDatabaseHighWaterBytes?: number; + readonly durableDatabaseStartBytes?: number; + readonly durableFilesystemBytes?: number; + readonly durableFilesystemHighWaterBytes?: number; + readonly durableJournalBytes?: number; + readonly durableJournalHighWaterBytes?: number; + readonly durableSharedMemoryBytes?: number; + readonly durableSharedMemoryHighWaterBytes?: number; + readonly durableWalBytes?: number; + readonly durableWalHighWaterBytes?: number; + readonly estimateBasis?: 'cached-fact-bytes' | 'final-fact-bytes' | 'source-bytes-fallback'; + readonly estimatedConcurrentBuildBytes?: number; + readonly estimatedDurableFilesystemRequiredBytes?: number; + readonly estimatedDurableSnapshotBytes?: number; + readonly estimatedJournalBytes?: number; + readonly estimatedRequiredBytes?: number; + readonly estimatedTemporaryFilesystemRequiredBytes?: number; + readonly estimatedTemporaryDatabaseBytes?: number; + readonly filesystemsShared?: boolean; + readonly materializationMode?: 'direct-persistent' | 'temporary-staged'; + readonly temporaryAvailableBytes?: number; + readonly temporaryDatabaseBytes: number; + readonly temporaryDatabaseHighWaterBytes: number; +} + +export function graphBuildIsActive(build: GraphBuildStatus): boolean { + return ( + (build.state === 'queued' || build.state === 'running') && + build.observation.liveness === 'active' && + build.coordination?.role !== 'history' + ); +} + +export function graphBuildShouldDisplay(build: GraphBuildStatus): boolean { + return build.state === 'failed' || graphBuildIsActive(build); +} + +export const GRAPH_ADMINISTRATION_JOB_LIMIT = 4; + +export interface GraphAdministrationJobSelection { + readonly hiddenCount: number; + readonly jobs: readonly GraphBuildStatus[]; + readonly total: number; +} + +/** Keep administration cards focused on bounded, actionable build state. */ +export function graphAdministrationJobSelection( + builds: readonly GraphBuildStatus[], + waiters: readonly GraphBuildStatus[], +): GraphAdministrationJobSelection { + const unique = new Map(); + for (const job of [...builds, ...waiters]) { + if (graphBuildShouldDisplay(job) && !unique.has(job.buildId)) unique.set(job.buildId, job); + } + const relevant = [...unique.values()].sort(compareGraphAdministrationJob); + const jobs = relevant.slice(0, GRAPH_ADMINISTRATION_JOB_LIMIT); + return {hiddenCount: relevant.length - jobs.length, jobs, total: relevant.length}; +} + +export function compareGraphAdministrationJob(left: GraphBuildStatus, right: GraphBuildStatus): number { + const priority = (job: GraphBuildStatus) => (job.state === 'running' ? 0 : job.state === 'queued' ? 1 : 2); + return ( + priority(left) - priority(right) || + (Date.parse(right.timestamps.lastProgressAt) || 0) - (Date.parse(left.timestamps.lastProgressAt) || 0) || + compareCodeUnits(left.buildId, right.buildId) + ); +} + +export function graphAdministrationInventorySummary( + summary: Pick, +): string { + return [ + graphAdministrationCount(summary.databaseCount, 'graph database'), + graphAdministrationCount(summary.readySnapshotCount, 'stored ready snapshot'), + graphAdministrationCount(summary.viewCount, 'active worktree view'), + ].join(' · '); +} + +export function graphAdministrationCount(count: number, singular: string): string { + return `${count.toLocaleString()} ${singular}${count === 1 ? '' : 's'}`; +} + +export interface GraphBuildTarget { + readonly repositoryLabel: string; + readonly worktreeLabel: string; +} + +export interface GraphBuildConcurrencyState { + readonly activeTargetCommit?: string; + readonly latestTargetCommit: string; + readonly queuedRequests: number; + readonly readySnapshotCommit?: string; + readonly staleReady: boolean; +} + +export function graphBuildTarget( + build: GraphBuildStatus, + repositories: readonly GraphRepositoryGroup[], +): GraphBuildTarget { + const repository = repositories.find(candidate => candidate.repositoryId === build.identity.repositoryId); + const view = repository?.views.find( + candidate => + candidate.checkoutId === build.identity.checkoutId && candidate.worktreeId === build.identity.worktreeId, + ); + const fallbackName = build.identity.displayName?.trim(); + const repositoryLabel = repository + ? graphRepositoryOptionLabel(repository, repositories) + : fallbackName + ? fallbackName + : 'Indexed repository'; + const folder = view?.localAssociation.displayPath ?? build.managerContext?.worktreePath; + const branch = view?.localAssociation.branch + ? `observed worktree branch ${view.localAssociation.branch}` + : build.managerContext?.branch + ? `build-start branch ${build.managerContext.branch}` + : undefined; + return { + repositoryLabel, + worktreeLabel: + ([branch, folder].filter((value): value is string => value !== undefined).join(' · ') || view?.label) ?? + `Local folder unavailable · commit ${build.identity.commit.slice(0, 8) || 'unknown'}`, + }; +} + +/** + * Summarize only observed concurrency facts. File locks do not promise FIFO, so + * waiters are counted without claiming an execution position. The most recent + * request is the latest requested target, independent of input ordering. + */ +export function graphBuildConcurrencyState( + build: GraphBuildStatus, + waiters: readonly GraphBuildStatus[], + repositories: readonly GraphRepositoryGroup[], +): GraphBuildConcurrencyState { + const matchingWaiters = waiters.filter( + waiter => + waiter.buildId !== build.buildId && + waiter.identity.checkoutId === build.identity.checkoutId && + waiter.identity.worktreeId === build.identity.worktreeId, + ); + const latest = [build, ...matchingWaiters].sort(compareGraphBuildRequest)[matchingWaiters.length]!; + const repository = repositories.find(candidate => candidate.repositoryId === build.identity.repositoryId); + const ready = repository?.views.find( + candidate => + candidate.checkoutId === build.identity.checkoutId && candidate.worktreeId === build.identity.worktreeId, + ); + const queuedRequests = matchingWaiters.length + (build.state === 'queued' ? 1 : 0); + const readySnapshotCommit = ready?.snapshot.commit; + return { + ...(build.state === 'running' ? {activeTargetCommit: build.identity.commit} : {}), + latestTargetCommit: latest.identity.commit, + queuedRequests, + ...(readySnapshotCommit === undefined ? {} : {readySnapshotCommit}), + staleReady: readySnapshotCommit !== undefined && !graphCommitMatches(readySnapshotCommit, latest.identity.commit), + }; +} + +export function compareGraphBuildRequest(left: GraphBuildStatus, right: GraphBuildStatus): number { + const leftStartedAt = Date.parse(left.timestamps.startedAt) || 0; + const rightStartedAt = Date.parse(right.timestamps.startedAt) || 0; + return leftStartedAt - rightStartedAt || compareCodeUnits(left.buildId, right.buildId); +} + +export function graphCommitMatches(left: string, right: string): boolean { + return left === right || left.startsWith(right) || right.startsWith(left); +} + +export function graphStatusPollDelay( + builds: readonly GraphBuildStatus[], + maintenance?: CodeGraphMaintenanceStatus, + lifecyclePending = false, + automaticCompaction?: CodeGraphAutomaticCompactionStatus, +): number { + return builds.some(graphBuildIsActive) || + maintenance !== undefined || + lifecyclePending || + automaticCompaction?.state === 'inspecting' || + automaticCompaction?.state === 'running' + ? 1_000 + : 5_000; +} + +/** Keep live build activity visible even when the full catalog request has not completed. */ +export function mergeGraphCatalogStatus( + catalog: GraphCatalog | undefined, + status: Pick< + GraphCatalog, + | 'automaticCompaction' + | 'builds' + | 'catalogRevision' + | 'lifecyclePending' + | 'maintenance' + | 'storage' + | 'waiterCount' + | 'waiters' + >, +): GraphCatalog { + const base = catalog ?? { + builds: [], + diagnostics: [], + repositories: [], + waiterCount: 0, + waiters: [], + }; + const {maintenance: _previousMaintenance, ...catalogWithoutMaintenance} = base; + return { + ...catalogWithoutMaintenance, + ...status, + ...(status.catalogRevision === undefined && base.catalogRevision !== undefined + ? {catalogRevision: base.catalogRevision} + : {}), + ...(status.storage === undefined && base.storage !== undefined ? {storage: base.storage} : {}), + ...(status.automaticCompaction === undefined && base.automaticCompaction !== undefined + ? {automaticCompaction: base.automaticCompaction} + : {}), + }; +} + +export type GraphStorageSummary = ManagerGraphStorageSummary; + +export function graphMaintenanceStatusLabel(status: CodeGraphMaintenanceStatus): string { + const operation = status.operation === 'selected-snapshot-purge' ? 'Selected snapshot purge' : 'Graph maintenance'; + const phases: Record = { + 'acquiring-gates': 'acquiring safety gates', + 'retiring-and-cleaning': 'retiring snapshot and advancing cleanup', + 'status-unavailable': 'working; detailed status unavailable', + 'verifying-graph': 'rechecking graph safety evidence', + 'verifying-vectors': 'rechecking vector safety evidence', + 'waiting-builders': 'waiting for graph builders', + working: 'working', + }; + return `${operation} · ${phases[status.phase]}`; +} + +export function graphCompletedBuildResultIdentity(build: GraphBuildStatus): string | undefined { + return build.state === 'completed' && build.result !== undefined + ? `${build.buildId}:${build.result.snapshotId}` + : undefined; +} + +export function graphStatusRequiresCatalogRefresh( + catalog: GraphCatalog | undefined, + builds: readonly GraphBuildStatus[], + acknowledgedResults: ReadonlySet = new Set(), + observedCatalogRevision?: string, +): boolean { + if ( + catalog !== undefined && + observedCatalogRevision !== undefined && + catalog.catalogRevision !== observedCatalogRevision + ) { + return true; + } + if (!catalog) { + return builds.some(build => { + const identity = graphCompletedBuildResultIdentity(build); + return identity !== undefined && !acknowledgedResults.has(identity); + }); + } + return builds.some(build => { + const identity = graphCompletedBuildResultIdentity(build); + const resultVisible = catalog.repositories.some( + repository => + repository.repositoryId === build.identity.repositoryId && + repository.views.some( + view => + view.checkoutId === build.identity.checkoutId && + view.worktreeId === build.identity.worktreeId && + view.snapshot.id === build.result?.snapshotId, + ), + ); + return identity !== undefined && !acknowledgedResults.has(identity) && !resultVisible; + }); +} + +export function graphDiagnosticsRequiresCatalogRefresh( + diagnosticsCatalogRevision: string | undefined, + observedCatalogRevision: string | undefined, + maintenance?: CodeGraphMaintenanceStatus, +): boolean { + return ( + maintenance === undefined && + observedCatalogRevision !== undefined && + diagnosticsCatalogRevision !== observedCatalogRevision + ); +} + +export function graphWaiterCountForBuild(build: GraphBuildStatus, waiters: readonly GraphBuildStatus[]): number { + return waiters.filter( + waiter => + waiter.identity.checkoutId === build.identity.checkoutId && + waiter.identity.worktreeId === build.identity.worktreeId && + waiter.request?.key === build.request?.key, + ).length; +} + +export function resolveGraphSelection( + repositories: readonly GraphRepositoryGroup[], + currentRepositoryId: string, + currentViewId: string, +): {readonly repositoryId: string; readonly viewId: string} { + const repository = repositories.find(candidate => candidate.id === currentRepositoryId) ?? repositories[0]; + if (!repository) return {repositoryId: '', viewId: ''}; + const view = repository.views.find(candidate => candidate.id === currentViewId); + return { + repositoryId: repository.id, + viewId: view?.id ?? repository.defaultViewId ?? repository.views[0]?.id ?? '', + }; +} + +export function graphRepositoryOptionLabel( + repository: GraphRepositoryGroup, + repositories: readonly GraphRepositoryGroup[], +): string { + const collides = repositories.some( + candidate => candidate.id !== repository.id && candidate.displayName === repository.displayName, + ); + if (!collides) return repository.displayName; + const folder = repository.views.find(view => view.localAssociation.displayPath)?.localAssociation.displayPath; + return `${repository.displayName} · ${folder ?? `repository ${repository.id.slice(0, 8)}`}`; +} + +export function shortGraphIdentity(value: string): string { + return value.slice(-8) || 'unknown'; +} + +export function mergeGraphRepositoryGroups( + current: readonly GraphRepositoryGroup[], + additions: readonly GraphRepositoryGroup[], +): readonly GraphRepositoryGroup[] { + const groups = new Map(current.map(group => [group.id, {...group, views: [...group.views]}])); + for (const addition of additions) { + const existing = groups.get(addition.id); + if (!existing) { + groups.set(addition.id, {...addition, views: [...addition.views]}); + continue; + } + const views = new Map(existing.views.map(view => [view.id, view])); + for (const view of addition.views) { + const currentView = views.get(view.id); + views.set(view.id, currentView ? mergeGraphRepository(currentView, view) : view); + } + groups.set(addition.id, { + ...existing, + defaultViewId: existing.defaultViewId || addition.defaultViewId, + views: [...views.values()], + viewsTruncated: existing.viewsTruncated || addition.viewsTruncated, + }); + } + return [...groups.values()].sort( + (left, right) => compareCodeUnits(left.displayName, right.displayName) || compareCodeUnits(left.id, right.id), + ); +} + +export function graphCatalogPageOffsets(input: { + readonly baseRepository?: GraphRepository; + readonly baseRepositoryGroup?: GraphRepositoryGroup; + readonly checkoutId: string; + readonly continuation?: { + readonly projectOffset: number; + readonly viewId: string; + readonly viewOffset: number; + readonly workspaceOffset: number; + }; + readonly viewId: string; +}): {readonly projectOffset: number; readonly viewOffset: number; readonly workspaceOffset: number} { + const continuation = input.continuation?.viewId === input.viewId ? input.continuation : undefined; + return { + projectOffset: + continuation?.projectOffset ?? + input.baseRepository?.projects.filter(project => project.id.startsWith('cgp_')).length ?? + 0, + viewOffset: + continuation?.viewOffset ?? + input.baseRepositoryGroup?.views.filter(view => view.checkoutId === input.checkoutId).length ?? + 0, + workspaceOffset: continuation?.workspaceOffset ?? input.baseRepository?.workspaces.length ?? 0, + }; +} + +export function mergeGraphRepository(current: GraphRepository, addition: GraphRepository): GraphRepository { + if (current.snapshot.id !== addition.snapshot.id) { + const currentTime = Date.parse(current.activatedAt ?? current.snapshot.completedAt ?? '') || 0; + const additionTime = Date.parse(addition.activatedAt ?? addition.snapshot.completedAt ?? '') || 0; + return additionTime > currentTime ? addition : current; + } + const projects = new Map(current.projects.map(project => [project.id, project])); + for (const project of addition.projects) projects.set(project.id, project); + const workspaces = new Map(current.workspaces.map(workspace => [workspace.id, workspace])); + for (const workspace of addition.workspaces) workspaces.set(workspace.id, workspace); + return { + ...current, + ...addition, + projectCount: Math.max(current.projectCount, addition.projectCount), + projects: [...projects.values()], + projectsTruncated: current.projectsTruncated || addition.projectsTruncated, + workspaceCount: Math.max(current.workspaceCount, addition.workspaceCount), + workspaces: [...workspaces.values()], + workspacesTruncated: current.workspacesTruncated || addition.workspacesTruncated, + }; +} + +export interface GraphNode { + readonly degree: number; + readonly exported?: boolean; + readonly fileCount?: number; + readonly id: string; + readonly kind: string; + readonly label: string; + readonly language?: string; + readonly packageName?: string; + readonly path?: string; + readonly projectId: string; + readonly qualifiedName?: string; + readonly signature?: string; + readonly symbolCount?: number; + readonly type: 'project' | 'symbol'; +} + +export interface GraphEdge { + readonly confidence: number; + readonly count: number; + readonly id: string; + readonly provenance: string; + readonly relation: string; + readonly sourceId: string; + readonly targetId: string; +} + +export interface GraphSpan { + readonly column: number; + readonly endColumn: number; + readonly endLine: number; + readonly line: number; +} + +export interface GraphNodeDetail { + readonly node: { + readonly documentation?: string; + readonly exported: boolean; + readonly id: string; + readonly kind: string; + readonly label: string; + readonly language: string; + readonly packageName?: string; + readonly path: string; + readonly projectId: string; + readonly qualifiedName: string; + readonly signature?: string; + readonly span: GraphSpan; + }; + readonly relationships: readonly { + readonly confidence: number; + readonly direction: 'incoming' | 'outgoing'; + readonly evidencePath: string; + readonly evidenceSpan: GraphSpan; + readonly id: string; + readonly provenance: string; + readonly related: { + readonly id?: string; + readonly kind?: string; + readonly label: string; + readonly path?: string; + readonly projectId?: string; + readonly qualifiedName?: string; + }; + readonly relation: string; + }[]; + readonly snapshotId: string; + readonly stats: { + readonly incoming: number; + readonly outgoing: number; + readonly sampledEdges?: number; + readonly summaryTruncated?: boolean; + readonly provenances: readonly {readonly count: number; readonly provenance: string}[]; + readonly relations: readonly { + readonly count: number; + readonly incoming: number; + readonly outgoing: number; + readonly relation: string; + }[]; + readonly truncated: boolean; + }; +} + +export function graphRelationshipCountLabel(count: number, sampled: boolean): string { + return `${sampled ? '≥' : ''}${Math.max(0, count).toLocaleString()}`; +} + +export function graphRelationshipSampleLabel(detail: GraphNodeDetail): string | undefined { + if (detail.stats.summaryTruncated !== true) return undefined; + const sampledEdges = detail.stats.sampledEdges ?? detail.stats.incoming + detail.stats.outgoing; + return `Counts are lower bounds from a ${sampledEdges.toLocaleString()}-edge sample.`; +} + +export function graphDisplayEdges( + edges: readonly GraphEdge[], + selectedNodeId: string | undefined, + focusMode: GraphFocusMode, + relationFilter: string, +): readonly GraphEdge[] { + const related = relationFilter === 'all' ? edges : edges.filter(edge => edge.relation === relationFilter); + if (!selectedNodeId || focusMode === 'all') return related; + return related.filter(edge => { + if (focusMode === 'incoming') return edge.targetId === selectedNodeId; + if (focusMode === 'outgoing') return edge.sourceId === selectedNodeId; + return edge.sourceId === selectedNodeId || edge.targetId === selectedNodeId; + }); +} + +export function graphAnalysisRequestIsCurrent( + currentSequence: number, + requestedSequence: number, + currentScope: string, + requestedScope: string, +): boolean { + return currentSequence === requestedSequence && currentScope === requestedScope; +} + +export function graphRequestIsCurrent( + currentSequence: number, + requestedSequence: number, + currentScope: string, + requestedScope: string, +): boolean { + return currentSequence === requestedSequence && currentScope === requestedScope; +} + +export function graphQueryRequestIsCurrent( + aborted: boolean, + currentSequence: number, + requestedSequence: number, + currentScope: string, + requestedScope: string, + graph: GraphQueryVisualization, + expectedSnapshotId: string, + expectedQuery: string, +): boolean { + return ( + !aborted && + graphRequestIsCurrent(currentSequence, requestedSequence, currentScope, requestedScope) && + graph.repository.snapshot.id === expectedSnapshotId && + graph.query.state === 'ready' && + graph.query.text.trim() === expectedQuery + ); +} + +export interface GraphQueryRequestInput { + readonly expectedQuery: string; + readonly expectedSnapshotId: string; + readonly scope: string; +} + +export type GraphQueryRequestOutcome = + | {readonly graph: GraphQueryVisualization; readonly state: 'accepted'} + | {readonly cause: unknown; readonly state: 'failed'} + | {readonly state: 'cancelled'} + | {readonly graph: GraphQueryVisualization; readonly state: 'stale'}; + +export interface GraphQueryRequestHandle { + readonly cancel: () => void; + readonly isCurrent: () => boolean; + readonly result: Promise; +} + +export interface GraphQueryRequestGate { + readonly cancelCurrent: () => void; + readonly request: ( + input: GraphQueryRequestInput, + load: (signal: AbortSignal) => Promise, + ) => GraphQueryRequestHandle; +} + +/** + * Owns the same supersession boundary used by the Manager graph-query UI. + * + * A new request aborts the previous signal. The sequence, scope, snapshot, and + * query checks remain mandatory even when a loader ignores cancellation and + * eventually delivers a late response. + */ +export function createGraphQueryRequestGate(): GraphQueryRequestGate { + let currentController: AbortController | undefined; + let currentScope = ''; + let currentSequence = 0; + + const cancelCurrent = (): void => { + currentController?.abort(); + currentController = undefined; + currentScope = ''; + currentSequence += 1; + }; + + return { + cancelCurrent, + request: (input, load) => { + currentController?.abort(); + const controller = new AbortController(); + const requestedSequence = currentSequence + 1; + currentController = controller; + currentScope = input.scope; + currentSequence = requestedSequence; + const isCurrent = (): boolean => + currentController === controller && + graphRequestIsCurrent(currentSequence, requestedSequence, currentScope, input.scope); + const cancel = (): void => { + if (!isCurrent()) return; + cancelCurrent(); + }; + let pending: Promise; + try { + pending = load(controller.signal); + } catch (cause) { + pending = Promise.reject(cause); + } + const result = pending.then( + graph => + graphQueryRequestIsCurrent( + controller.signal.aborted, + currentSequence, + requestedSequence, + currentScope, + input.scope, + graph, + input.expectedSnapshotId, + input.expectedQuery, + ) + ? {graph, state: 'accepted'} + : {graph, state: 'stale'}, + cause => + controller.signal.aborted || isAbortError(cause) + ? {state: 'cancelled'} + : isCurrent() + ? {cause, state: 'failed'} + : {state: 'cancelled'}, + ); + return {cancel, isCurrent, result}; + }, + }; +} + +export function graphNodeDetailRequestIsCurrent( + aborted: boolean, + detail: Pick, + expectedSnapshotId: string, + expectedNodeId: string, +): boolean { + return !aborted && detail.snapshotId === expectedSnapshotId && detail.node.id === expectedNodeId; +} + +export function cacheGraphNodeDetail( + cache: Map, + key: string, + detail: GraphNodeDetail, + limit = 128, +): void { + cache.delete(key); + cache.set(key, detail); + while (cache.size > Math.max(1, limit)) { + const oldest = cache.keys().next().value as string | undefined; + if (oldest === undefined) break; + cache.delete(oldest); + } +} + +export function graphWithNodeNeighborhood(graph: GraphVisualization, detail: GraphNodeDetail): GraphVisualization { + if (graph.mode !== 'detail') return graph; + const nodesById = new Map(graph.nodes.slice(0, MANAGER_GRAPH_MAX_NODE_LIMIT).map(node => [node.id, node])); + const existingRoot = nodesById.get(detail.node.id); + if (existingRoot || nodesById.size < MANAGER_GRAPH_MAX_NODE_LIMIT) { + nodesById.set(detail.node.id, { + ...existingRoot, + degree: existingRoot?.degree ?? 0, + exported: detail.node.exported, + id: detail.node.id, + kind: detail.node.kind, + label: detail.node.label, + language: detail.node.language, + packageName: detail.node.packageName, + path: detail.node.path, + projectId: detail.node.projectId, + qualifiedName: detail.node.qualifiedName, + signature: detail.node.signature, + type: 'symbol', + }); + } + + const edgesById = new Map(graph.edges.slice(0, MANAGER_GRAPH_MAX_EDGE_LIMIT).map(edge => [edge.id, edge])); + let truncated = graph.nodes.length > nodesById.size || graph.edges.length > edgesById.size; + for (const relationship of detail.relationships.slice(0, MAX_EXPANDED_NEIGHBOR_EDGES)) { + const relatedId = relationship.related.id; + if (!relatedId || relatedId === detail.node.id) continue; + if (!nodesById.has(relatedId)) { + if (nodesById.size >= MANAGER_GRAPH_MAX_NODE_LIMIT) { + truncated = true; + continue; + } + nodesById.set(relatedId, { + degree: 0, + id: relatedId, + kind: relationship.related.kind ?? 'symbol', + label: relationship.related.label, + path: relationship.related.path, + projectId: relationship.related.projectId ?? detail.node.projectId, + qualifiedName: relationship.related.qualifiedName, + type: 'symbol', + }); + } + if (!edgesById.has(relationship.id)) { + if (edgesById.size >= MANAGER_GRAPH_MAX_EDGE_LIMIT || !nodesById.has(detail.node.id)) { + truncated = true; + continue; + } + const outgoing = relationship.direction === 'outgoing'; + edgesById.set(relationship.id, { + confidence: relationship.confidence, + count: 1, + id: relationship.id, + provenance: relationship.provenance, + relation: relationship.relation, + sourceId: outgoing ? detail.node.id : relatedId, + targetId: outgoing ? relatedId : detail.node.id, + }); + } + } + + const edges = [...edgesById.values()]; + const degrees = graphNodeSizeValues(edges, 'connections'); + const nodes = [...nodesById.values()].map(node => ({...node, degree: degrees.get(node.id) ?? 0})); + const addedNodes = nodes.length - graph.nodes.length; + return { + ...graph, + edges, + nodes, + stats: { + ...graph.stats, + renderedEdges: edges.length, + renderedNodes: nodes.length, + }, + paging: {...graph.paging, hasMore: graph.paging.hasMore || truncated || detail.stats.truncated}, + warnings: [ + ...graph.warnings, + ...(addedNodes > 0 ? [`Loaded ${addedNodes.toLocaleString()} direct neighbors for ${detail.node.label}.`] : []), + ...(truncated ? ['Direct-neighbor expansion reached the global Manager graph budget.'] : []), + ], + }; +} + +export interface GraphVisualization { + readonly edges: readonly GraphEdge[]; + readonly mode: 'detail' | 'overview'; + readonly nodes: readonly GraphNode[]; + readonly paging: { + readonly edgeLimit: number; + readonly hasMore: boolean; + readonly nodeLimit: number; + }; + readonly projectId: string; + readonly query?: GraphQueryMetadata; + readonly repository: Pick; + readonly scope: {readonly id: string; readonly label: string}; + readonly stats: { + readonly renderedEdges: number; + readonly renderedNodes: number; + readonly totalEdges: number; + readonly totalNodes: number; + }; + readonly warnings: readonly string[]; +} + +export interface GraphQueryMetadata { + readonly matchedNodes: number; + readonly state: 'ready'; + readonly text: string; + readonly warnings: readonly string[]; +} + +export interface GraphQueryVisualization extends GraphVisualization { + readonly query: GraphQueryMetadata; +} + +export interface GraphCatalogContinuation { + readonly projectOffset: number; + readonly projectHasMore: boolean; + readonly viewOffset: number; + readonly viewHasMore: boolean; + readonly viewId: string; + readonly workspaceOffset: number; + readonly workspaceHasMore: boolean; +} + +export interface GraphCatalogSearchOptions { + readonly projects: readonly { + readonly description: string; + readonly id: string; + readonly label: string; + readonly viewId: string; + }[]; + readonly views: readonly { + readonly description: string; + readonly id: string; + readonly label: string; + readonly repositoryId: string; + }[]; +} + +export function graphCatalogSearchOptions( + repository: GraphRepository, + repositories: readonly GraphRepositoryGroup[], +): GraphCatalogSearchOptions { + const workspaces = new Map(repository.workspaces.map(workspace => [workspace.id, workspace])); + const projects = repository.projects + .map(project => { + const workspace = project.workspaceId ? workspaces.get(project.workspaceId) : undefined; + return { + description: [workspace?.name, graphProjectBadge(project)].filter(Boolean).join(' · '), + id: project.id, + label: project.label, + viewId: repository.id, + }; + }) + .sort((left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id)); + const viewsById = new Map< + string, + {readonly description: string; readonly id: string; readonly label: string; readonly repositoryId: string} + >(); + for (const group of repositories) { + for (const view of group.views) { + viewsById.set(view.id, { + description: `${group.displayName} · ${view.snapshot.commit.slice(0, 8)}${view.snapshot.dirty ? ' · dirty' : ''}${view.localAssociation.branch ? ` · observed branch ${view.localAssociation.branch}` : ''} · folder ${graphLocalAssociationText(view.localAssociation)}`, + id: view.id, + label: view.label, + repositoryId: group.id, + }); + } + } + return { + projects, + views: [...viewsById.values()].sort( + (left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id), + ), + }; +} + +export function graphCatalogContinuationHasMore( + continuation: GraphCatalogContinuation | undefined, + viewId: string | undefined, + field: 'projectHasMore' | 'viewHasMore' | 'workspaceHasMore', + fallback: boolean, +): boolean { + return continuation !== undefined && continuation.viewId === viewId ? continuation[field] : fallback; +} + +export interface GraphAnalysis { + readonly communities: readonly { + readonly id: string; + readonly label: string; + readonly memberCount: number; + }[]; + readonly coverage: { + readonly complete: boolean; + readonly topology: { + readonly complete: boolean; + readonly state: 'complete' | 'not-requested' | 'partial' | 'unavailable'; + }; + }; + readonly hubs: readonly { + readonly classification: 'god-node' | 'hub'; + readonly degree: number; + readonly node: {readonly label: string; readonly path: string}; + }[]; + readonly statistics: { + readonly analyzedEdgeCount: number; + readonly analyzedNodeCount: number; + readonly communityCount: number; + readonly connectedComponentCount: number; + readonly maximumDegree: number; + }; + readonly surprisingLinks: readonly { + readonly relation: string; + readonly score: number; + readonly source: {readonly label: string}; + readonly target: {readonly label: string}; + }[]; + readonly warnings: readonly string[]; +} + +export function graphAnalysisTopologyAvailable(analysis: GraphAnalysis): boolean { + return analysis.coverage.topology.state === 'complete' || analysis.coverage.topology.state === 'partial'; +} + +export function graphAnalysisCoverageLabel(analysis: GraphAnalysis): string { + switch (analysis.coverage.topology.state) { + case 'complete': + return analysis.coverage.complete ? 'Complete' : 'Topology complete'; + case 'partial': + return 'Topology partial'; + case 'not-requested': + return 'Topology not requested'; + case 'unavailable': + return 'Topology unavailable'; + } +} + +export interface PositionedNode extends GraphNode { + readonly color: THREE.Color; + readonly radius: number; + readonly x: number; + readonly y: number; +} + +export interface GraphLayout { + readonly bounds: {readonly height: number; readonly width: number}; + readonly nodes: readonly PositionedNode[]; + readonly nodesById: ReadonlyMap; +} + +export interface GraphPosition { + readonly x: number; + readonly y: number; +} + +export interface GraphLabelSize { + readonly height: number; + readonly width: number; +} + +export interface GraphRuntime { + readonly camera: THREE.OrthographicCamera; + readonly edgePosition: THREE.BufferAttribute; + readonly edges: readonly GraphEdge[]; + readonly highlightPosition?: THREE.BufferAttribute; + readonly highlightedEdges: readonly GraphEdge[]; + readonly nodeIds: readonly string[]; + readonly nodePosition: THREE.BufferAttribute; + readonly pointMaterials: readonly THREE.ShaderMaterial[]; + readonly renderer: THREE.WebGLRenderer; + readonly scene: THREE.Scene; + readonly selectedNodeId?: string; + readonly selectedPosition?: THREE.BufferAttribute; +} + +export interface ViewState { + readonly x: number; + readonly y: number; + readonly zoom: number; +} + +export type GraphFocusMode = 'all' | 'incoming' | 'neighbors' | 'outgoing'; +export type GraphSizeMetric = 'connections' | 'incoming' | 'outgoing'; + +export const GRAPH_PALETTE = ['#67e8c7', '#7aa2ff', '#c08cff', '#ff9f7a', '#f7d56b', '#75d8ff', '#ef88b7', '#9be27d']; +export const SELECTED_NODE_COLOR = '#ff4fd8'; +export const MIN_ZOOM = 0.32; +export const MAX_ZOOM = 8; +export const DEFAULT_WORKING_SET = { + edgeLimit: MANAGER_GRAPH_DEFAULT_EDGE_LIMIT, + nodeLimit: MANAGER_GRAPH_DEFAULT_NODE_LIMIT, +} as const; +export const MAX_WORKING_SET = { + edgeLimit: MANAGER_GRAPH_MAX_EDGE_LIMIT, + nodeLimit: MANAGER_GRAPH_MAX_NODE_LIMIT, +} as const; +export const MAX_ANIMATED_NEIGHBOR_EDGES = 120; +export const MAX_EXPANDED_NEIGHBOR_EDGES = 160; +export const MAX_FOCUSED_LABELS = 24; +export const FOCUS_LAYOUT_ZOOM = 2.8; +export const SEARCH_FOCUS_ZOOM = { + detail: 2.8, + overview: 1.8, +} as const; +export const GRAPH_QUERY_DEBOUNCE_MILLISECONDS = 450; +export const GRAPH_QUERY_MINIMUM_LENGTH = 3; +export const GRAPH_QUERY_MAXIMUM_LENGTH = 512; +export const DEFAULT_QUERY_WORKING_SET = {edgeLimit: 240, nodeLimit: 120} as const; +export const MAX_QUERY_WORKING_SET = {edgeLimit: 500, nodeLimit: 200} as const; + +export function managerGraphQueryCandidate(input: string): string | undefined { + const candidate = input.trim(); + return candidate.length > 0 && candidate.length <= GRAPH_QUERY_MAXIMUM_LENGTH ? candidate : undefined; +} + +export function managerGraphDebouncedQueryCandidate(input: string): string | undefined { + const candidate = managerGraphQueryCandidate(input); + return candidate && candidate.length >= GRAPH_QUERY_MINIMUM_LENGTH ? candidate : undefined; +} + +export function graphOverviewSizeLabel(graph: GraphVisualization): string { + return graph.repository.metrics === 'complete' && graph.nodes.some(node => node.symbolCount !== undefined) + ? 'Component symbols' + : 'Visible relationship degree'; +} +export function graphLocalAssociationText(association: CodeGraphLocalAssociation): string { + return association.displayPath ?? association.state.replaceAll('-', ' '); +} +export function graphNodeSizeValues( + edges: readonly Pick[], + metric: GraphSizeMetric, +): ReadonlyMap { + const connected = new Map>(); + const add = (nodeId: string, neighborId: string): void => { + const neighbors = connected.get(nodeId) ?? new Set(); + neighbors.add(neighborId); + connected.set(nodeId, neighbors); + }; + for (const edge of edges) { + if (edge.sourceId === edge.targetId) continue; + if (metric !== 'incoming') add(edge.sourceId, edge.targetId); + if (metric !== 'outgoing') add(edge.targetId, edge.sourceId); + } + return new Map([...connected].map(([nodeId, neighbors]) => [nodeId, neighbors.size])); +} +export function isAbortError(cause: unknown): boolean { + return cause instanceof DOMException + ? cause.name === 'AbortError' + : cause instanceof Error && cause.name === 'AbortError'; +} + +export function compactNumber(value: number): string { + return new Intl.NumberFormat(undefined, {maximumFractionDigits: 1, notation: 'compact'}).format(value); +} + +export function graphProjectBadge(project: GraphProject): string { + if (project.model === 'legacy-fallback') return 'legacy group'; + if (project.model === 'facet') return 'facet'; + return project.buildSystem ? `${project.buildSystem} ${project.kind ?? 'component'}` : (project.kind ?? 'component'); +} + +export function relationLabel(value: string): string { + return value.replaceAll('_', ' ').replace(/\b\w/g, letter => letter.toUpperCase()); +} + +export function sizeMetricLabel(metric: GraphSizeMetric): string { + switch (metric) { + case 'incoming': + return 'Distinct incoming neighbors'; + case 'outgoing': + return 'Distinct outgoing neighbors'; + default: + return 'Distinct connections'; + } +} + +export function sourceBreadcrumb(projectId: string, path: string): readonly string[] { + const project = projectId.replace(/^[^:]+:/, ''); + const parts = path.split('/').filter(Boolean); + const compactPath = parts.length > 4 ? [...parts.slice(0, 2), '…', ...parts.slice(-2)] : parts; + return [project, ...compactPath.filter((part, index) => index > 0 || part !== project)]; +} diff --git a/src/manager_graph_panels.tsx b/src/manager_graph_panels.tsx new file mode 100644 index 00000000..b1ea0004 --- /dev/null +++ b/src/manager_graph_panels.tsx @@ -0,0 +1,1486 @@ +import React, {useEffect, useState} from 'react'; +import type {CodeGraphAutomaticCompactionStatus} from './code_graph/automatic_compaction.js'; +import type {CodeGraphLocalDiagnosticsReport} from './code_graph/diagnostics.js'; +import type {CodeGraphMaintenanceStatus} from './code_graph/maintenance_gate.js'; +import { + CODE_GRAPH_SLOW_FILE_THRESHOLD_MILLISECONDS, + CODE_GRAPH_TOP_SLOW_FILE_LIMIT, +} from './code_graph/progress_telemetry.js'; +import { + compactNumber, + graphAdministrationInventorySummary, + graphAdministrationJobSelection, + graphAdministrationTarget, + graphAnalysisCoverageLabel, + graphAnalysisTopologyAvailable, + graphBuildConcurrencyState, + graphBuildTarget, + graphLocalAssociationText, + graphMaintenanceStatusLabel, + graphRelationshipCountLabel, + graphRelationshipSampleLabel, + graphViewRemovalTarget, + graphWaiterCountForBuild, + GRAPH_PALETTE, + relationLabel, + SELECTED_NODE_COLOR, + shortGraphIdentity, + sizeMetricLabel, + sourceBreadcrumb, + type GraphAdministrationAction, + type GraphAnalysis, + type GraphBuildStatus, + type GraphEdge, + type GraphMaterializationRows, + type GraphMaterializationStage, + type GraphMaterializationStorage, + type GraphNode, + type GraphNodeDetail, + type GraphRepositoryGroup, + type GraphSizeMetric, + type GraphStorageSummary, + type GraphVisualization, + type GraphWorktreeAdministrationAction, +} from './manager_graph_model.js'; +import {type ManagerDialogOptions, useOptionalManagerDialogs} from './manager_dialog.js'; + +export function GraphSummary(props: { + readonly analysis?: GraphAnalysis; + readonly analysisError: string; + readonly analysisLoading: boolean; + readonly graph: GraphVisualization; + readonly onAnalyze: () => void; + readonly sizeMetric: GraphSizeMetric; +}): React.ReactElement { + return ( +
+

{props.graph.mode === 'overview' ? 'Repository overview' : 'Component working set'}

+

{props.graph.scope.label}

+

+ {props.graph.mode === 'overview' + ? props.graph.repository.metrics === 'complete' + ? 'Node size reflects indexed symbol volume. Double-click a component to explore its symbol graph.' + : 'Node size reflects visible relationship degree. Double-click a component to explore its symbol graph.' + : `Node size reflects ${sizeMetricLabel(props.sizeMetric).toLowerCase()} among the filtered relationships.`} +

+
+
+
Indexed symbols
+
{compactNumber(props.graph.stats.totalNodes)}
+
+
+
Visible nodes
+
{compactNumber(props.graph.stats.renderedNodes)}
+
+
+
Visible links
+
{compactNumber(props.graph.stats.renderedEdges)}
+
+
+
Snapshot
+
+ {props.graph.repository.snapshot.commit.slice(0, 8)} + {props.graph.repository.snapshot.dirty ? ' + dirty' : ''} +
+
+ {props.graph.mode === 'overview' ? ( +
+
Overview coverage
+
+ {props.graph.repository.metrics === 'deferred' + ? 'Computed on demand' + : `${compactNumber(props.graph.repository.accounting.attributedSymbols)} / ${compactNumber( + props.graph.repository.accounting.totalSymbols, + )}`} +
+
+ ) : null} +
+
+ + Component or facet + + + Selected node + + + Size ·{' '} + {props.graph.mode === 'overview' + ? props.graph.repository.metrics === 'complete' + ? 'Component symbols' + : 'Visible relationships' + : sizeMetricLabel(props.sizeMetric)} + +
+
+
+
+

Whole-graph analysis

+

Architecture signals

+
+ +
+ {props.analysis ? ( + <> +
+
+
Communities
+
+ {graphAnalysisTopologyAvailable(props.analysis) + ? compactNumber(props.analysis.statistics.communityCount) + : 'Unavailable'} +
+
+
+
Components
+
+ {graphAnalysisTopologyAvailable(props.analysis) + ? compactNumber(props.analysis.statistics.connectedComponentCount) + : 'Unavailable'} +
+
+
+
Hubs
+
+ {graphAnalysisTopologyAvailable(props.analysis) + ? compactNumber(props.analysis.hubs.length) + : 'Unavailable'} +
+
+
+
Coverage
+
{graphAnalysisCoverageLabel(props.analysis)}
+
+
+ {props.analysis.hubs.length > 0 ? ( +
+
Highest-connectivity nodes
+ {props.analysis.hubs.slice(0, 4).map(hub => ( +
+ + {hub.node.label} + {hub.node.path} + + + {hub.classification === 'god-node' ? 'God node' : 'Hub'} · {hub.degree} + +
+ ))} +
+ ) : graphAnalysisTopologyAvailable(props.analysis) ? null : ( +

Topology was not derived, so community, component, and hub absence is not inferred.

+ )} + {props.analysis.surprisingLinks[0] ? ( +

+ Cross-community signal: {props.analysis.surprisingLinks[0].source.label}{' '} + {relationLabel(props.analysis.surprisingLinks[0].relation)}{' '} + {props.analysis.surprisingLinks[0].target.label} +

+ ) : null} + {props.analysis.warnings.length > 0 ?

{props.analysis.warnings[0]}

: null} + + ) : props.analysisError ? ( +

{props.analysisError}

+ ) : ( +

Run deterministic communities, hub, and cross-boundary analysis on demand.

+ )} +
+
+ ); +} + +export function NodeInspector(props: { + readonly detail?: GraphNodeDetail; + readonly detailError: string; + readonly detailLoading: boolean; + readonly graph: GraphVisualization; + readonly node: GraphNode; + readonly onOpenProject: () => void; + readonly onSelectNode: (nodeId: string) => void; +}): React.ReactElement { + const [tab, setTab] = useState<'evidence' | 'overview' | 'relationships'>('overview'); + useEffect(() => setTab('overview'), [props.node.id]); + const connected = props.graph.edges.filter( + edge => edge.sourceId === props.node.id || edge.targetId === props.node.id, + ); + const nodesById = new Map(props.graph.nodes.map(node => [node.id, node])); + const localRelated = connected + .slice() + .sort((left, right) => right.count - left.count || right.confidence - left.confidence) + .slice(0, 7) + .map(edge => { + const id = edge.sourceId === props.node.id ? edge.targetId : edge.sourceId; + return {edge, node: nodesById.get(id)}; + }) + .filter((item): item is {readonly edge: GraphEdge; readonly node: GraphNode} => item.node !== undefined); + const visibleNodeIds = new Set(props.graph.nodes.map(node => node.id)); + const detail = props.detail?.node.id === props.node.id ? props.detail : undefined; + const sourceLocation = detail + ? `${detail.node.path}:${detail.node.span.line}:${detail.node.span.column}` + : props.node.path; + const breadcrumb = detail ? sourceBreadcrumb(detail.node.projectId, detail.node.path) : []; + const relationshipCountsSampled = detail?.stats.summaryTruncated === true; + const relationshipSampleLabel = detail ? graphRelationshipSampleLabel(detail) : undefined; + return ( +
+
+
+ {props.node.kind} + {props.node.exported ? exported : null} + {props.node.projectId !== props.graph.projectId && props.graph.mode === 'detail' ? ( + context + ) : null} +
+

{props.node.label}

+

{props.node.qualifiedName ?? props.node.projectId.replace(/^[^:]+:/, '')}

+ {breadcrumb.length > 0 ? ( +
+ {breadcrumb.map((part, index) => ( + + {index > 0 ? : null} + {part} + + ))} +
+ ) : null} +
+ {props.node.type === 'project' ? ( + + ) : ( +
+ {( + [ + ['overview', 'Overview'], + ['relationships', 'Relations'], + ['evidence', 'Evidence'], + ] as const + ).map(([value, label]) => ( + + ))} +
+ )} + + {props.detailLoading ? ( +
+
+ ) : null} + {props.detailError ? ( +
+ Detailed evidence unavailable: {props.detailError} +
+ ) : null} + + {props.node.type === 'project' || tab === 'overview' ? ( + <> + {detail?.node.documentation ?

{detail.node.documentation}

: null} +
+ {sourceLocation ? ( + <> +
Source
+
{sourceLocation}
+ + ) : null} + {props.node.language ? ( + <> +
Language
+
{props.node.language}
+ + ) : null} + {detail?.node.packageName ? ( + <> +
Package
+
{detail.node.packageName}
+ + ) : null} +
{detail ? 'Fan-in' : 'Visible degree'}
+
+ {detail + ? graphRelationshipCountLabel(detail.stats.incoming, relationshipCountsSampled) + : props.node.degree.toLocaleString()} +
+ {detail ? ( + <> +
Fan-out
+
{graphRelationshipCountLabel(detail.stats.outgoing, relationshipCountsSampled)}
+ + ) : null} + {props.node.symbolCount !== undefined ? ( + <> +
Symbols
+
{props.node.symbolCount.toLocaleString()}
+
Files
+
{props.node.fileCount?.toLocaleString()}
+ + ) : null} +
+ {detail?.stats.provenances.length ? ( +
+ {detail.stats.provenances.map(item => ( + + {item.provenance}{' '} + {graphRelationshipCountLabel(item.count, relationshipCountsSampled)} + + ))} +
+ ) : null} + {relationshipSampleLabel ?

{relationshipSampleLabel}

: null} + {(detail?.node.signature ?? props.node.signature) ? ( + 
{detail?.node.signature ?? props.node.signature}
+ ) : null} + {props.node.type === 'project' && localRelated.length ? ( +
+

Strongest visible links

+ {localRelated.map(({edge, node}) => ( + + ))} +
+ ) : null} + + ) : null} + + {props.node.type === 'symbol' && tab === 'relationships' ? ( + detail ? ( +
+
+ + {graphRelationshipCountLabel(detail.stats.incoming, relationshipCountsSampled)}{' '} + incoming + + + {graphRelationshipCountLabel(detail.stats.outgoing, relationshipCountsSampled)}{' '} + outgoing + +
+
+ {detail.stats.relations.map(item => { + const maximum = detail.stats.relations[0]?.count ?? 1; + return ( +
+ + {relationLabel(item.relation)} + + {graphRelationshipCountLabel(item.incoming, relationshipCountsSampled)} in ·{' '} + {graphRelationshipCountLabel(item.outgoing, relationshipCountsSampled)} out + + + +
+ ); + })} +
+ {relationshipSampleLabel ?

{relationshipSampleLabel}

: null} +
+

Direct neighborhood

+ {detail.relationships.slice(0, 32).map(relationship => { + const canSelect = Boolean(relationship.related.id && visibleNodeIds.has(relationship.related.id)); + return ( + + ); + })} +
+ {detail.stats.truncated ? ( +

Showing the strongest 160 relationships from this node.

+ ) : null} +
+ ) : ( +

Relationship details are not available.

+ ) + ) : null} + + {props.node.type === 'symbol' && tab === 'evidence' ? ( + detail?.relationships.length ? ( +
+ {detail.relationships.slice(0, 32).map(relationship => ( +
+
+ {relationLabel(relationship.relation)} + {Math.round(relationship.confidence * 100)}% +
+

+ {relationship.direction === 'incoming' ? 'From' : 'To'}{' '} + {relationship.related.qualifiedName ?? relationship.related.label} +

+ + {relationship.evidencePath}:{relationship.evidenceSpan.line}:{relationship.evidenceSpan.column} + +
+ {relationship.provenance} + + lines {relationship.evidenceSpan.line}–{relationship.evidenceSpan.endLine} + +
+
+ ))} + {detail.stats.truncated ? ( +

+ Evidence is capped at 160 relationships to keep inspection responsive. +

+ ) : null} +
+ ) : ( +

No relationship evidence is indexed for this node.

+ ) + ) : null} +
+ ); +} + +export function GraphAdministration(props: { + readonly busy?: string; + readonly onAction: (action: GraphAdministrationAction) => void; + readonly onDiagnostics: (options: {readonly analyze: boolean; readonly deep: boolean}) => void; + readonly output?: string; + readonly report?: CodeGraphLocalDiagnosticsReport; +}): React.ReactElement { + const dialogs = useOptionalManagerDialogs(); + const [analyze, setAnalyze] = useState(false); + const [deep, setDeep] = useState(false); + const [forceCompact, setForceCompact] = useState(false); + const blocked = props.busy !== undefined; + const confirmAction = async (options: ManagerDialogOptions, action: GraphAdministrationAction): Promise => { + if (await dialogs.confirm(options)) props.onAction(action); + }; + const targetAction = async ( + managementAvailable: boolean, + action: GraphWorktreeAdministrationAction, + ): Promise => { + if (managementAvailable) return action; + const values = await dialogs.prompt({ + confirmLabel: 'Use worktree', + detail: 'The local folder is not currently associated with this indexed view.', + fields: [ + { + description: 'Threadnote verifies this path against the indexed checkout and worktree before acting.', + id: 'cwd', + label: 'Absolute worktree path', + placeholder: '/absolute/path/to/worktree', + required: true, + }, + ], + message: 'Threadnote has no current local path for this indexed view.', + title: 'Locate the indexed worktree', + }); + return values ? {...action, cwd: values.cwd} : undefined; + }; + const dispatchTargetAction = async ( + managementAvailable: boolean, + action: GraphWorktreeAdministrationAction, + confirmation?: ManagerDialogOptions, + ): Promise => { + const targeted = await targetAction(managementAvailable, action); + if (!targeted) return; + if (confirmation && !(await dialogs.confirm(confirmation))) return; + props.onAction(targeted); + }; + return ( +
+ + + Graph administration + + {props.report + ? graphAdministrationInventorySummary(props.report.summary) + : 'Load home-wide status, diagnostics, and maintenance controls'} + + + {props.busy ? {props.busy}… : null} + +
+
+ + + + + + + +
+ + {props.report ? ( +
+ {props.report.databases.map(database => { + const view = database.views.find(candidate => candidate.managementAvailable) ?? database.views[0]; + const managementAvailable = view?.managementAvailable === true; + const repository = view?.repository.displayName ?? 'Indexed repository'; + const jobs = graphAdministrationJobSelection(database.builds, database.waiters); + const obsolete = props.report?.obsoleteStores.checkouts.find( + checkout => checkout.checkoutId === database.checkoutId, + ); + const target = view + ? graphAdministrationTarget(database.checkoutId, { + repository: view.repository, + worktreeId: view.viewWorktreeId, + }) + : undefined; + const health = database.health?.integrity ?? database.healthState; + return ( +
+
+ + {repository} + + {view?.localAssociation.branch ? `observed branch ${view.localAssociation.branch} · ` : ''} + {view?.localAssociation.displayPath ?? + 'Local folder unavailable; opaque ID shown in diagnostics'} + + + {health === 'migration-pending' ? 'migrating' : health} +
+
+
+
Stored ready snapshots
+
+ {database.health + ? database.health.readySnapshots.toLocaleString() + : database.healthState === 'deferred' + ? 'health inspection deferred' + : 'unavailable'} +
+
+
+
Active worktree views
+
{database.views.length.toLocaleString()}
+
+
+
Storage
+
+ {database.storage.state === 'available' + ? `${formatGraphBytes(database.storage.filesystemBytes)} physical DB + sidecars` + : 'missing'} +
+
+
+
Jobs
+
{jobs.total === 0 ? 'None' : `${jobs.total} actionable`}
+
+
+

+ Snapshot and view counts can differ: views are per-worktree pointers, while ready snapshots are + stored graph versions that can be shared, retained for reuse, or protected while in use. +

+ {database.storage.state === 'available' && 'pageStorage' in database.storage ? ( + database.storage.pageStorage.state === 'available' ? ( +

+ SQLite pages:{' '} + {formatGraphBytes( + database.storage.pageStorage.pageCount * database.storage.pageStorage.pageSize - + database.storage.pageStorage.reclaimableBytes, + )}{' '} + in use · {formatGraphBytes(database.storage.pageStorage.reclaimableBytes)} already reusable + inside the file + {database.storage.pageStorage.compactionOpportunityBytes === undefined || + database.storage.pageStorage.compactionOpportunityBytes === + database.storage.pageStorage.reclaimableBytes + ? '' + : ` · ${formatGraphBytes( + database.storage.pageStorage.compactionOpportunityBytes, + )} total compaction opportunity`} + {database.storage.pageStorage.threshold.reason === 'freelist' + ? ' · eligible for automatic compaction after retry-cooldown, disk, and maintenance safety checks' + : database.storage.pageStorage.threshold.reason === 'freelist-and-fragmentation' + ? ' · manual compaction opportunity; automatic compaction is withheld for live-page structural slack' + : ''} +

+ ) : database.storage.pageStorage.state === 'deferred' ? ( +

+ SQLite page usage is deferred while an active build owns this repository. Manager will retry + after the build releases its lock. +

+ ) : ( +

+ SQLite page usage and automatic-compaction eligibility could not be established. Run graph + diagnostics before retrying repair or compaction. +

+ ) + ) : null} +
+ {database.views.map(candidate => { + const removalTarget = graphViewRemovalTarget(database.checkoutId, { + snapshot: candidate.snapshot, + worktreeId: candidate.viewWorktreeId, + }); + return ( +
+ {candidate.repository.displayName} + + {candidate.snapshot.fileCount.toLocaleString()} files ·{' '} + {candidate.snapshot.symbolCount.toLocaleString()} symbols ·{' '} + {candidate.snapshot.edgeCount.toLocaleString()} edges + + + {candidate.localAssociation.branch + ? `Observed branch ${candidate.localAssociation.branch} · ` + : ''} + Folder: {graphLocalAssociationText(candidate.localAssociation)} ·{' '} + {candidate.localAssociation.state} + + {candidate.analysis ? ( + + {candidate.analysis.coverage.complete ? 'Complete' : 'Partial'} analysis ·{' '} + {candidate.analysis.coverage.topology.state === 'complete' || + candidate.analysis.coverage.topology.state === 'partial' ? ( + <> + {candidate.analysis.statistics.connectedComponentCount.toLocaleString()} components ·{' '} + {candidate.analysis.statistics.communityCount.toLocaleString()} communities · average + degree {candidate.analysis.statistics.averageDegree.toFixed(2)} · maximum{' '} + {candidate.analysis.statistics.maximumDegree.toLocaleString()} ·{' '} + {candidate.analysis.statistics.isolatedNodeCount.toLocaleString()} isolated + + ) : ( + <>topology {candidate.analysis.coverage.topology.state} + )} + + ) : null} + +
+ ); + })} +
+ {jobs.jobs.map(job => { + const jobView = database.views.find( + candidate => candidate.viewWorktreeId === job.identity.worktreeId, + ); + return ( +

+ {jobView?.repository.displayName ?? 'Indexed repository'} + {jobView ? ` · folder ${graphLocalAssociationText(jobView.localAssociation)}` : ''} ·{' '} + {job.state === 'running' ? 'active' : job.state} · {job.phase} + {job.subphase ? `/${job.subphase}` : ''} · {job.observation.liveness} + {job.error ? ` · ${job.error.summary}` : ''} +

+ ); + })} + {jobs.hiddenCount > 0 ? ( +

+{jobs.hiddenCount} more active or failed jobs

+ ) : null} + {database.issues.map(issue => ( +

+ {issue.code}: {issue.message} +

+ ))} +
+ + + + + {obsolete ? ( + <> + + + + ) : null} + + +
+ {!managementAvailable ? ( + + Index, reindex, and compact require a verified local worktree path. Purge actions target this + inventoried checkout directly. + + ) : null} +
+ ); + })} +
+ ) : ( +

Load diagnostics to enumerate every local graph database.

+ )} + {props.output ? 
{props.output}
: null} +
+
+ ); +} + +export function GraphMaintenanceProgress(props: { + readonly repositories: readonly GraphRepositoryGroup[]; + readonly status: CodeGraphMaintenanceStatus; +}): React.ReactElement { + const {status} = props; + const repository = props.repositories + .flatMap(group => group.views) + .find(view => view.checkoutId === status.checkoutId); + const elapsed = status.startedAt === undefined ? undefined : Math.max(0, Date.now() - Date.parse(status.startedAt)); + const lastUpdate = + status.updatedAt === undefined ? undefined : Math.max(0, Date.now() - Date.parse(status.updatedAt)); + const percentage = + status.completed !== undefined && status.total !== undefined && status.total > 0 + ? Math.max(0, Math.min(100, (status.completed / status.total) * 100)) + : undefined; + return ( +
+
+
+
+ + {status.operation === 'selected-snapshot-purge' ? 'Selected snapshot purge' : 'Graph maintenance'} + + + {status.checkoutId + ? repository + ? `${repository.displayName} · folder ${graphLocalAssociationText(repository.localAssociation)}` + : 'Indexed repository' + : 'Home-wide maintenance'} + +
+ {elapsed === undefined ? null : Elapsed {formatBuildDuration(elapsed)}} +
+

{graphMaintenanceStatusLabel(status)}

+ {percentage === undefined ? null : ( +
+ +
+ )} +

+ {status.completed === undefined || status.total === undefined + ? 'Waiting for the next maintenance phase update' + : `${status.completed.toLocaleString()} / ${status.total.toLocaleString()} safety phases`} + {lastUpdate === undefined ? '' : ` · last update ${formatBuildDuration(lastUpdate)} ago`} +

+
+
+ ); +} + +export function GraphAutomaticCompactionProgress(props: { + readonly repositories: readonly GraphRepositoryGroup[]; + readonly status: CodeGraphAutomaticCompactionStatus; +}): React.ReactElement | null { + const {status} = props; + if (status.state === 'idle') return null; + const checkoutId = 'checkoutId' in status ? status.checkoutId : undefined; + const repository = props.repositories.flatMap(group => group.views).find(view => view.checkoutId === checkoutId); + const target = repository + ? `${repository.displayName} · folder ${graphLocalAssociationText(repository.localAssociation)}` + : checkoutId + ? 'Indexed repository' + : 'Local graph storage'; + const message = (() => { + switch (status.state) { + case 'inspecting': + return 'Checking bounded graph storage receipts for a safe reclaim opportunity.'; + case 'running': + return `Compacting ${target} in an isolated process without blocking Manager.`; + case 'deferred': + return `Compaction for ${target} was deferred because ${ + status.reason === 'active-build' ? 'a graph build is active' : 'another maintenance operation is active' + }. Manager will retry.`; + case 'failed': + return status.reason === 'inspection-failed' + ? 'The automatic storage check could not inspect any candidate. Review graph diagnostics.' + : 'The isolated compaction result could not be confirmed. Review diagnostics before retrying.'; + case 'completed': + return status.action === 'compacted' + ? `Compacted ${target} and returned ${formatGraphBytes(status.reclaimedBytes)} to the filesystem.` + : `Automatic storage check completed: no eligible graph required compaction (${status.inspected.toLocaleString()} inspected${ + status.inspectionFailures === 0 ? '' : `, ${status.inspectionFailures.toLocaleString()} unavailable` + }).`; + } + })(); + return ( +
+
+
+
+ Automatic graph storage compaction + {target} +
+
+

{message}

+
+
+ ); +} + +export function GraphBuildProgress(props: { + readonly build: GraphBuildStatus; + readonly repositories: readonly GraphRepositoryGroup[]; + readonly storage?: GraphStorageSummary; + readonly waiters: readonly GraphBuildStatus[]; +}): React.ReactElement { + const {build} = props; + const completed = build.counters.completed; + const total = build.counters.total; + const percentage = + completed !== undefined && total !== undefined && total > 0 + ? Math.max(0, Math.min(100, (completed / total) * 100)) + : undefined; + const elapsed = Math.max(0, Date.now() - Date.parse(build.timestamps.startedAt)); + const lastProgress = Math.max(0, Date.now() - Date.parse(build.timestamps.lastProgressAt)); + const progressSilent = build.coordination?.progressSilent === true; + const eta = progressSilent ? undefined : build.eta; + const target = graphBuildTarget(build, props.repositories); + const concurrency = graphBuildConcurrencyState(build, props.waiters, props.repositories); + const waiterCount = graphWaiterCountForBuild(build, props.waiters); + const statusLabel = + build.state === 'failed' + ? 'Indexing failed' + : build.state === 'queued' + ? 'Waiting to index' + : progressSilent + ? 'Indexing status is stale' + : 'Indexing'; + return ( +
+
+
+ {target.repositoryLabel} + {target.worktreeLabel} +
+ Elapsed {formatBuildDuration(elapsed)} +
+

+ {statusLabel} · {build.phase}/{build.subphase ?? 'working'} · commit {build.identity.commit} +

+

+ {build.state === 'running' + ? `Active target ${graphCommitLabel(build.identity.commit)}` + : build.state === 'queued' + ? `Queued target ${graphCommitLabel(build.identity.commit)}` + : build.state === 'failed' + ? `Failed target ${graphCommitLabel(build.identity.commit)}` + : `Completed target ${graphCommitLabel(build.identity.commit)}`} + {concurrency.latestTargetCommit === build.identity.commit + ? '' + : ` · latest target ${graphCommitLabel(concurrency.latestTargetCommit)} queued`} + {concurrency.queuedRequests === 0 + ? '' + : ` · ${concurrency.queuedRequests.toLocaleString()} queued request${concurrency.queuedRequests === 1 ? '' : 's'}`} +

+ {concurrency.staleReady && concurrency.readySnapshotCommit !== undefined ? ( +

+ Ready snapshot {graphCommitLabel(concurrency.readySnapshotCommit)} remains queryable · stale for latest target{' '} + {graphCommitLabel(concurrency.latestTargetCommit)} +

+ ) : null} + {percentage === undefined ? null : ( +
+ +
+ )} +

+ {build.phase === 'reclaiming' + ? `${(completed ?? 0).toLocaleString()} / ${(total ?? 0).toLocaleString()} snapshots · ${( + build.counters.pagesCompleted ?? 0 + ).toLocaleString()} pages · ${(build.counters.rowsDeleted ?? 0).toLocaleString()} rows reclaimed` + : completed === undefined || total === undefined + ? 'Preparing phase counters' + : `${completed.toLocaleString()} / ${total.toLocaleString()} ${build.counters.unit ?? 'items'}`} + {' · '}last progress change {formatBuildDuration(lastProgress)} ago +

+ {progressSilent ? ( +

+ No progress update for {formatBuildDuration(lastProgress)}. Process {build.owner.processId} still owns the + build lock, but Manager cannot determine whether its current operation is advancing. +

+ ) : null} + {props.storage?.state === 'available' ? ( + props.storage.pageStorage.state === 'available' ? ( +

+ Storage now: {formatGraphBytes(props.storage.physicalBytes)} physical SQLite file + sidecars ·{' '} + {formatGraphBytes(props.storage.pageStorage.inUseBytes)} pages in use ·{' '} + {formatGraphBytes(props.storage.pageStorage.reusableBytes)} reusable inside SQLite + {props.storage.pageStorage.compactionOpportunityBytes === undefined || + props.storage.pageStorage.compactionOpportunityBytes === props.storage.pageStorage.reusableBytes + ? '' + : ` · ${formatGraphBytes(props.storage.pageStorage.compactionOpportunityBytes)} total compaction opportunity`} + {props.storage.pageStorage.automaticCompaction === 'eligible' + ? ' · eligible for automatic compaction after retry-cooldown and maintenance safety checks' + : props.storage.pageStorage.automaticCompaction === 'waiting-for-space' + ? ` · automatic compaction is waiting for ${formatGraphBytes( + props.storage.pageStorage.requiredFreeBytes ?? 0, + )} free disk space` + : props.storage.pageStorage.automaticCompaction === 'space-unknown' + ? ' · free disk space could not be verified, so automatic compaction is withheld' + : ''} +

+ ) : ( +

+ Storage now: {formatGraphBytes(props.storage.physicalBytes)} physical SQLite file + sidecars · page usage + and automatic compaction decision deferred while an active build or database lock owns this checkout +

+ ) + ) : null} + {build.activity ? ( +

+ Current reported step: {build.activity.stage} {build.activity.language} ·{' '} + {formatGraphBytes(build.activity.bytes)} · batch {build.activity.batchCompleted.toLocaleString()}/ + {build.activity.batchTotal.toLocaleString()} + {build.activity.sizeBucket === undefined ? '' : ` · ${build.activity.sizeBucket} source bucket`} + {build.activity.role === undefined ? '' : ` · ${build.activity.role}`} + {build.activity.classifier === undefined ? '' : `/${build.activity.classifier}`} + {build.activity.factsBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.activity.factsBytes)} emitted facts`} + {build.activity.symbols === undefined ? '' : ` · ${build.activity.symbols.toLocaleString()} symbols`} + {build.activity.relations === undefined ? '' : ` · ${build.activity.relations.toLocaleString()} relations`} + {build.activity.parseMilliseconds === undefined + ? '' + : ` · parse ${formatGraphMilliseconds(build.activity.parseMilliseconds)}`} + {build.activity.persistMilliseconds === undefined + ? '' + : ` · persist ${formatGraphMilliseconds(build.activity.persistMilliseconds)}`} + {build.activity.degraded ? ' · metadata fallback; retry scheduled' : ''} +

+ ) : null} + {build.extraction ? ( +

+ Extraction telemetry: {build.extraction.completedFiles.toLocaleString()} files completed ·{' '} + {build.extraction.metrics === undefined + ? '' + : `${formatGraphBytes(build.extraction.metrics.sourceBytesCompleted)}/${formatGraphBytes( + build.extraction.metrics.sourceBytesTotal, + )} source · ${formatGraphBytes(build.extraction.metrics.factsBytesCompleted)} emitted facts · ${formatGraphPercentage( + build.extraction.metrics.workUnitsCompleted, + build.extraction.metrics.workUnitsTotal, + )} class-weighted work · `} + {build.extraction.slowFiles.toLocaleString()} at or above{' '} + {formatGraphMilliseconds(CODE_GRAPH_SLOW_FILE_THRESHOLD_MILLISECONDS)} · bounded top-slow evidence{' '} + {build.extraction.topSlowFiles.length.toLocaleString()}/{CODE_GRAPH_TOP_SLOW_FILE_LIMIT.toLocaleString()} +

+ ) : null} + {build.materialization?.metrics?.mode === 'full' ? ( +

+ Full materialization selected + {build.materialization.metrics.fallbackReason === undefined + ? '' + : ` · incremental fallback: ${build.materialization.metrics.fallbackReason.replaceAll('-', ' ')}`} +

+ ) : null} + {build.materialization?.activity ? ( +

+ Current reported step: {graphMaterializationStageLabel(build.materialization.activity.stage)} · batch{' '} + {graphActiveBatchNumber( + build.materialization.activity.batchCompleted, + build.materialization.activity.batchTotal, + ).toLocaleString()} + /{build.materialization.activity.batchTotal.toLocaleString()} ·{' '} + {formatGraphBytes(build.materialization.activity.sourceBytes)} source + {build.materialization.activity.cachedFactBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.activity.cachedFactBytes)} cached facts`} + {build.materialization.activity.factsBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.activity.factsBytes)} final facts`} + {graphMaterializationRows(build.materialization.activity.rows)} + {' · '}this step{' '} + {formatBuildDuration(Math.max(0, Date.now() - Date.parse(build.materialization.activity.startedAt)))} + {build.materialization.activity.transactionMilliseconds === undefined + ? '' + : ` · transaction ${formatGraphMilliseconds(build.materialization.activity.transactionMilliseconds)}`} +

+ ) : null} + {build.activation?.activity ? ( +

+ Current reported step: activating · {build.activation.activity.stage.replaceAll('-', ' ')} ·{' '} + {build.activation.activity.state} + {build.activation.activity.rows === undefined + ? '' + : ` · ${build.activation.activity.rows.toLocaleString()} rows`} + {' · '}stage {formatGraphMilliseconds(build.activation.activity.stageElapsedMilliseconds)} · total{' '} + {formatGraphMilliseconds(build.activation.activity.elapsedMilliseconds)} + {build.activation.activity.transactionMilliseconds === undefined + ? '' + : ` · transaction ${formatGraphMilliseconds(build.activation.activity.transactionMilliseconds)}`} +

+ ) : null} + {build.resolution?.activity ? ( +

+ Reference resolution: pass {build.resolution.activity.pass.toLocaleString()} · page{' '} + {build.resolution.activity.pageCompleted.toLocaleString()}/ + {build.resolution.activity.pageTotal.toLocaleString()} ·{' '} + {build.resolution.activity.referencesCompleted.toLocaleString()}/ + {build.resolution.activity.referencesTotal.toLocaleString()} references ·{' '} + {build.resolution.activity.referencesExamined.toLocaleString()} cumulative examined ·{' '} + {build.resolution.activity.resolved.toLocaleString()} linked ·{' '} + {build.resolution.activity.aliasesDiscovered.toLocaleString()} aliases · match{' '} + {formatGraphMilliseconds(build.resolution.activity.matchingMilliseconds)} · transactions{' '} + {formatGraphMilliseconds(build.resolution.activity.transactionMilliseconds)} · total{' '} + {formatGraphMilliseconds(build.resolution.activity.elapsedMilliseconds)} +

+ ) : null} + {build.materialization?.metrics ? ( + <> +

+ Materialized: {build.materialization.metrics.batchesCompleted.toLocaleString()}/ + {build.materialization.metrics.batchesTotal.toLocaleString()} batches ·{' '} + {formatGraphBytes(build.materialization.metrics.sourceBytesCompleted)}/ + {formatGraphBytes(build.materialization.metrics.sourceBytesTotal)} source + {build.materialization.metrics.cachedFactBytesCompleted === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.cachedFactBytesCompleted)}${ + build.materialization.metrics.cachedFactBytesTotal === undefined + ? '' + : `/${formatGraphBytes(build.materialization.metrics.cachedFactBytesTotal)}` + } cached facts`} + {build.materialization.metrics.factsBytesCompleted === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.factsBytesCompleted)}${ + build.materialization.metrics.factsBytesTotal === undefined + ? '' + : `/${formatGraphBytes(build.materialization.metrics.factsBytesTotal)}` + } final facts`} + {graphMaterializationRows(build.materialization.metrics.rows)} + {build.materialization.metrics.loadingMilliseconds === undefined + ? '' + : ` · load ${formatGraphMilliseconds(build.materialization.metrics.loadingMilliseconds)}`} + {build.materialization.metrics.attributionMilliseconds === undefined + ? '' + : ` · attribute ${formatGraphMilliseconds(build.materialization.metrics.attributionMilliseconds)}`} + {build.materialization.metrics.transactionMilliseconds === undefined + ? '' + : ` · transactions ${formatGraphMilliseconds(build.materialization.metrics.transactionMilliseconds)}`} +

+ {build.materialization.metrics.storage ? ( + <> +

+ Storage: + {build.materialization.metrics.storage.durableDatabaseBytes === undefined + ? '' + : ` ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseBytes)} allocated SQLite pages observed during this build`} + {build.materialization.metrics.storage.durableDatabaseHighWaterBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseHighWaterBytes)} SQLite allocation high-water`} + {build.materialization.metrics.storage.durableDatabaseGrowthHighWaterBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableDatabaseGrowthHighWaterBytes)} main-database growth`} + {build.materialization.metrics.storage.durableFilesystemHighWaterBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableFilesystemHighWaterBytes)} DB + sidecars high-water`} + {build.materialization.metrics.storage.durableWalHighWaterBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableWalHighWaterBytes)} WAL high-water`} + {build.materialization.metrics.storage.durableJournalHighWaterBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableJournalHighWaterBytes)} rollback-journal high-water`} + {build.materialization.metrics.storage.durableDatabaseBytes === undefined ? '' : ' ·'}{' '} + {formatGraphBytes(build.materialization.metrics.storage.temporaryDatabaseBytes)} current TEMP database ·{' '} + {formatGraphBytes(build.materialization.metrics.storage.temporaryDatabaseHighWaterBytes)} TEMP database + high-water + {build.materialization.metrics.storage.estimatedRequiredBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedRequiredBytes)} combined estimate`} + {build.materialization.metrics.storage.estimatedTemporaryFilesystemRequiredBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedTemporaryFilesystemRequiredBytes)} TEMP-filesystem requirement`} + {build.materialization.metrics.storage.estimatedDurableFilesystemRequiredBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.estimatedDurableFilesystemRequiredBytes)} graph-filesystem requirement`} + {build.materialization.metrics.storage.temporaryAvailableBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.temporaryAvailableBytes)} available for TEMP`} + {build.materialization.metrics.storage.durableAvailableBytes === undefined + ? '' + : ` · ${formatGraphBytes(build.materialization.metrics.storage.durableAvailableBytes)} available for graph database`} + {build.materialization.metrics.storage.filesystemsShared === true ? ' · shared filesystem' : ''} + {build.materialization.metrics.storage.materializationMode === undefined + ? '' + : ` · ${build.materialization.metrics.storage.materializationMode.replaceAll('-', ' ')}`} + {build.materialization.metrics.storage.estimateBasis === undefined + ? '' + : ` · estimate from ${build.materialization.metrics.storage.estimateBasis.replaceAll('-', ' ')}`} + {' · '}rollback journals excluded from TEMP totals +

+ {graphMaterializationDiskWarning(build.materialization.metrics.storage) ? ( +

+ {graphMaterializationDiskWarning(build.materialization.metrics.storage)} Indexing continues with live + storage telemetry. +

+ ) : null} + + ) : null} + + ) : null} + {build.timings ? ( +

+ Phase: read {formatGraphMilliseconds(build.timings.readingMilliseconds)} · parse{' '} + {formatGraphMilliseconds(build.timings.extractionMilliseconds)} · persist{' '} + {formatGraphMilliseconds(build.timings.persistenceMilliseconds)} +

+ ) : null} +
+ + Process {build.owner.processId} + {build.owner.processStartIdentity + ? ` · owner instance ${shortGraphIdentity(build.owner.processStartIdentity)}` + : ''} + + {eta && eta.confidence !== 'low' ? ( + + Estimated time remaining in this phase: {formatBuildDuration(eta.remainingMilliseconds)} · {eta.confidence}{' '} + confidence + {eta.basis ? ` · ${graphEtaBasisLabel(eta.basis)}` : ''} + + ) : null} + {waiterCount > 0 ? {waiterCount} waiting process(es) for this exact target : null} + {build.error ? {build.error.summary} : null} +
+
+ ); +} + +function graphCommitLabel(commit: string): string { + return commit.slice(0, 12) || 'unknown'; +} + +function graphActiveBatchNumber(completed: number, total: number): number { + return total === 0 ? 0 : Math.min(total, completed + 1); +} + +function graphMaterializationStageLabel(stage: GraphMaterializationStage): string { + switch (stage) { + case 'loading-cache': + return 'loading cached facts'; + case 'attributing': + return 'attributing facts'; + case 'preparing-rows': + return 'preparing rows'; + case 'writing-analysis': + return 'writing analysis summary'; + case 'writing-symbols': + return 'writing symbols'; + case 'writing-lookups': + return 'writing lookup keys'; + case 'writing-terms': + return 'writing lexical terms'; + case 'writing-edges': + return 'writing relationships'; + case 'writing-references': + return 'writing references'; + case 'writing-receipt': + return 'recording resumable batch'; + case 'writing-candidates': + return 'writing reference candidates'; + case 'writing-facts': + return 'writing graph facts'; + case 'committing': + return 'committing batch'; + } +} + +function graphMaterializationRows(rows: GraphMaterializationRows | undefined): string { + if (!rows) return ''; + const values = [ + rows.symbols === undefined ? undefined : `${rows.symbols.toLocaleString()} symbols`, + rows.lookupKeys === undefined ? undefined : `${rows.lookupKeys.toLocaleString()} lookup keys`, + rows.terms === undefined ? undefined : `${rows.terms.toLocaleString()} terms`, + rows.edges === undefined ? undefined : `${rows.edges.toLocaleString()} relationships`, + rows.references === undefined ? undefined : `${rows.references.toLocaleString()} references`, + rows.referenceCandidates === undefined ? undefined : `${rows.referenceCandidates.toLocaleString()} candidates`, + rows.reexports === undefined ? undefined : `${rows.reexports.toLocaleString()} re-exports`, + rows.deduplicatedEdges === undefined || rows.deduplicatedEdges === 0 + ? undefined + : `${rows.deduplicatedEdges.toLocaleString()} repeated relationships collapsed`, + rows.deduplicatedReferences === undefined || rows.deduplicatedReferences === 0 + ? undefined + : `${rows.deduplicatedReferences.toLocaleString()} repeated resolution records collapsed`, + ].filter((value): value is string => value !== undefined); + return values.length > 0 ? ` · ${values.join(', ')}` : ''; +} + +function graphMaterializationDiskWarning(storage: GraphMaterializationStorage): string | undefined { + if ( + storage.filesystemsShared === true && + storage.availableBytes !== undefined && + storage.estimatedRequiredBytes !== undefined && + storage.availableBytes < storage.estimatedRequiredBytes + ) { + return 'Low disk: shared TEMP and graph storage is below the conservative combined estimate.'; + } + const scopes: string[] = []; + if ( + storage.temporaryAvailableBytes !== undefined && + storage.estimatedTemporaryFilesystemRequiredBytes !== undefined && + storage.temporaryAvailableBytes < storage.estimatedTemporaryFilesystemRequiredBytes + ) { + scopes.push('SQLite TEMP'); + } + if ( + storage.durableAvailableBytes !== undefined && + storage.estimatedDurableFilesystemRequiredBytes !== undefined && + storage.durableAvailableBytes < storage.estimatedDurableFilesystemRequiredBytes + ) { + scopes.push('graph database'); + } + return scopes.length === 0 ? undefined : `Low disk: ${scopes.join(' and ')} storage is below its estimate.`; +} + +function graphEtaBasisLabel( + basis: 'cached-fact-bytes' | 'extraction-work' | 'files' | 'final-fact-bytes' | 'source-bytes', +): string { + switch (basis) { + case 'cached-fact-bytes': + return 'cached-fact bytes'; + case 'final-fact-bytes': + return 'final attributed fact bytes'; + case 'source-bytes': + return 'source bytes'; + case 'extraction-work': + return 'class-weighted extraction work'; + case 'files': + return 'files'; + } +} + +function formatGraphPercentage(completed: number, total: number): string { + if (total <= 0) return '0%'; + return `${Math.min(100, Math.max(0, (completed / total) * 100)).toFixed(1)}%`; +} + +export function GraphEmptyState(props: {readonly building: boolean}): React.ReactElement { + return ( +
+
+ ); +} + +function formatBuildDuration(milliseconds: number): string { + if (!Number.isFinite(milliseconds)) return 'unknown'; + const seconds = Math.max(0, Math.floor(milliseconds / 1_000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${seconds % 60}s`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +function formatGraphMilliseconds(milliseconds: number): string { + if (!Number.isFinite(milliseconds) || milliseconds < 0) return 'unknown'; + if (milliseconds < 1) return '<1ms'; + if (milliseconds < 1_000) return `${Math.round(milliseconds)}ms`; + return `${(milliseconds / 1_000).toFixed(milliseconds >= 10_000 ? 1 : 2)}s`; +} + +function formatGraphBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes < 0) return 'unknown'; + if (bytes < 1_024) return `${Math.round(bytes)} B`; + const units = ['KiB', 'MiB', 'GiB', 'TiB']; + let value = bytes / 1_024; + let unit = units[0]!; + for (const candidate of units.slice(1)) { + if (value < 1_024) break; + value /= 1_024; + unit = candidate; + } + return `${value >= 10 ? value.toFixed(1) : value.toFixed(2)} ${unit}`; +} diff --git a/src/manager_graph_scene.tsx b/src/manager_graph_scene.tsx new file mode 100644 index 00000000..840c27b8 --- /dev/null +++ b/src/manager_graph_scene.tsx @@ -0,0 +1,1209 @@ +import React, {useEffect, useMemo, useRef, useState} from 'react'; +import * as THREE from 'three'; +import {compareCodeUnits} from './code_graph/ordering.js'; +import { + compactNumber, + FOCUS_LAYOUT_ZOOM, + graphDisplayEdges, + graphNodeSizeValues, + GRAPH_PALETTE, + MAX_ANIMATED_NEIGHBOR_EDGES, + MAX_FOCUSED_LABELS, + MAX_ZOOM, + MIN_ZOOM, + SEARCH_FOCUS_ZOOM, + SELECTED_NODE_COLOR, + type GraphEdge, + type GraphFocusMode, + type GraphLabelSize, + type GraphLayout, + type GraphNode, + type GraphPosition, + type GraphRuntime, + type GraphSizeMetric, + type GraphVisualization, + type PositionedNode, + type ViewState, +} from './manager_graph_model.js'; + +export function ThreeGraph(props: { + readonly focusRequest?: {readonly nodeId: string; readonly sequence: number}; + readonly focusMode: GraphFocusMode; + readonly graph: GraphVisualization; + readonly onOpenProject: (projectId: string) => void; + readonly onSelectNode: (nodeId: string | undefined) => void; + readonly relationFilter: string; + readonly selectedNodeId?: string; + readonly sizeMetric: GraphSizeMetric; +}): React.ReactElement { + const containerRef = useRef(null); + const canvasRef = useRef(null); + const dragRef = useRef<{moved: boolean; pointerId: number; x: number; y: number} | undefined>(undefined); + const labelRefs = useRef(new Map()); + const livePositionsRef = useRef>(new Map()); + const runtimeRef = useRef(undefined); + const [settledPositions, setSettledPositions] = useState>(() => new Map()); + const [size, setSize] = useState({height: 1, width: 1}); + const sizingEdges = useMemo( + () => + props.relationFilter === 'all' + ? props.graph.edges + : props.graph.edges.filter(edge => edge.relation === props.relationFilter), + [props.graph.edges, props.relationFilter], + ); + const baseLayout = useMemo( + () => buildGraphLayout(props.graph, props.sizeMetric, sizingEdges), + [props.graph, props.sizeMetric, sizingEdges], + ); + const layout = useMemo(() => graphLayoutWithPositions(baseLayout, settledPositions), [baseLayout, settledPositions]); + const displayEdges = useMemo( + () => graphDisplayEdges(props.graph.edges, props.selectedNodeId, props.focusMode, props.relationFilter), + [props.focusMode, props.graph.edges, props.relationFilter, props.selectedNodeId], + ); + const neighborhoodEdges = useMemo( + () => + props.selectedNodeId + ? displayEdges.filter(edge => edge.sourceId === props.selectedNodeId || edge.targetId === props.selectedNodeId) + : [], + [displayEdges, props.selectedNodeId], + ); + const animatedNeighborhoodEdges = useMemo( + () => neighborhoodEdges.slice(0, MAX_ANIMATED_NEIGHBOR_EDGES), + [neighborhoodEdges], + ); + const highlightedNodeIds = useMemo( + () => + props.selectedNodeId + ? new Set([props.selectedNodeId, ...animatedNeighborhoodEdges.flatMap(edge => [edge.sourceId, edge.targetId])]) + : undefined, + [animatedNeighborhoodEdges, props.selectedNodeId], + ); + const activeNodeIds = useMemo(() => { + if (!props.selectedNodeId || props.focusMode === 'all') return undefined; + return new Set([props.selectedNodeId, ...displayEdges.flatMap(edge => [edge.sourceId, edge.targetId])]); + }, [displayEdges, props.focusMode, props.selectedNodeId]); + const [view, setView] = useState(() => fittedView(layout, size)); + const viewRef = useRef(view); + const [focusLayoutRevision, setFocusLayoutRevision] = useState(0); + const [renderError, setRenderError] = useState(''); + + useEffect(() => { + setView(fittedView(layout, size)); + }, [props.graph.projectId, props.graph.repository.id, props.graph.repository.snapshot.id, size.height, size.width]); + + useEffect(() => { + viewRef.current = view; + }, [view]); + + useEffect(() => { + const request = props.focusRequest; + const node = request ? layout.nodesById.get(request.nodeId) : undefined; + if (!request || !node) return; + const startedAt = performance.now(); + const duration = 360; + const start = viewRef.current; + const target = graphFocusTarget(start, graphPosition(node, livePositionsRef.current), props.graph.mode); + let frame = 0; + const animate = (now: number): void => { + const progress = Math.min(1, (now - startedAt) / duration); + const eased = 1 - Math.pow(1 - progress, 3); + setView({ + x: lerp(start.x, target.x, eased), + y: lerp(start.y, target.y, eased), + zoom: lerp(start.zoom, target.zoom, eased), + }); + if (progress < 1) frame = window.requestAnimationFrame(animate); + else setFocusLayoutRevision(current => current + 1); + }; + frame = window.requestAnimationFrame(animate); + return () => window.cancelAnimationFrame(frame); + }, [props.focusRequest?.sequence, props.graph.mode]); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + const observer = new ResizeObserver(entries => { + const bounds = entries[0]?.contentRect; + if (bounds) setSize({height: Math.max(1, bounds.height), width: Math.max(1, bounds.width)}); + }); + observer.observe(container); + return () => observer.disconnect(); + }, []); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + let renderer: THREE.WebGLRenderer; + try { + renderer = new THREE.WebGLRenderer({ + alpha: true, + antialias: true, + canvas, + powerPreference: 'high-performance', + }); + setRenderError(''); + } catch { + setRenderError('WebGL is unavailable in this browser. Enable hardware acceleration to render the graph.'); + return; + } + renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); + renderer.setSize(size.width, size.height, false); + renderer.outputColorSpace = THREE.SRGBColorSpace; + const scene = new THREE.Scene(); + const camera = new THREE.OrthographicCamera(); + updateCamera(camera, view, size); + const currentPositions = livePositionsRef.current; + + const edgePositions: number[] = []; + const edgeColors: number[] = []; + const renderedEdges = displayEdges.filter( + edge => layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId), + ); + for (const edge of renderedEdges) { + const source = layout.nodesById.get(edge.sourceId); + const target = layout.nodesById.get(edge.targetId); + if (!source || !target) continue; + const sourcePosition = graphPosition(source, currentPositions); + const targetPosition = graphPosition(target, currentPositions); + edgePositions.push(sourcePosition.x, sourcePosition.y, 0, targetPosition.x, targetPosition.y, 0); + edgeColors.push(source.color.r, source.color.g, source.color.b, target.color.r, target.color.g, target.color.b); + } + const edgeGeometry = new THREE.BufferGeometry(); + const edgePosition = new THREE.Float32BufferAttribute(edgePositions, 3); + edgeGeometry.setAttribute('position', edgePosition); + edgeGeometry.setAttribute('color', new THREE.Float32BufferAttribute(edgeColors, 3)); + const edgeMaterial = new THREE.LineBasicMaterial({ + blending: THREE.AdditiveBlending, + opacity: props.graph.mode === 'overview' ? 0.34 : 0.18, + transparent: true, + vertexColors: true, + }); + const lines = new THREE.LineSegments(edgeGeometry, edgeMaterial); + scene.add(lines); + + const positions: number[] = []; + const colors: number[] = []; + const pointSizes: number[] = []; + for (const node of layout.nodes) { + const color = activeNodeIds && !activeNodeIds.has(node.id) ? node.color.clone().multiplyScalar(0.12) : node.color; + const position = graphPosition(node, currentPositions); + positions.push(position.x, position.y, 1); + colors.push(color.r, color.g, color.b); + pointSizes.push(node.radius * 2); + } + const nodeGeometry = new THREE.BufferGeometry(); + const nodePosition = new THREE.Float32BufferAttribute(positions, 3); + nodeGeometry.setAttribute('position', nodePosition); + nodeGeometry.setAttribute('color', new THREE.Float32BufferAttribute(colors, 3)); + nodeGeometry.setAttribute('pointSize', new THREE.Float32BufferAttribute(pointSizes, 1)); + const nodeMaterial = graphPointMaterial(1, view.zoom); + const points = new THREE.Points(nodeGeometry, nodeMaterial); + scene.add(points); + + const renderedHighlightedEdges = animatedNeighborhoodEdges.filter( + edge => layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId), + ); + const highlightPositions = directionalEdgePositions(renderedHighlightedEdges, layout.nodesById, currentPositions); + let highlightGeometry: THREE.BufferGeometry | undefined; + let highlightPosition: THREE.BufferAttribute | undefined; + let highlightMaterial: THREE.LineBasicMaterial | undefined; + if (highlightPositions.length > 0) { + highlightGeometry = new THREE.BufferGeometry(); + highlightPosition = new THREE.Float32BufferAttribute(highlightPositions, 3); + highlightGeometry.setAttribute('position', highlightPosition); + highlightMaterial = new THREE.LineBasicMaterial({ + blending: THREE.AdditiveBlending, + color: SELECTED_NODE_COLOR, + opacity: 0.72, + transparent: true, + }); + scene.add(new THREE.LineSegments(highlightGeometry, highlightMaterial)); + } + + const selectedNode = props.selectedNodeId ? layout.nodesById.get(props.selectedNodeId) : undefined; + let selectedGeometry: THREE.BufferGeometry | undefined; + let selectedPosition: THREE.BufferAttribute | undefined; + let selectedMaterial: THREE.ShaderMaterial | undefined; + if (selectedNode) { + const position = graphPosition(selectedNode, currentPositions); + selectedGeometry = new THREE.BufferGeometry(); + selectedPosition = new THREE.Float32BufferAttribute([position.x, position.y, 2], 3); + selectedGeometry.setAttribute('position', selectedPosition); + selectedGeometry.setAttribute( + 'color', + new THREE.Float32BufferAttribute(new THREE.Color(SELECTED_NODE_COLOR).toArray(), 3), + ); + selectedGeometry.setAttribute('pointSize', new THREE.Float32BufferAttribute([selectedNode.radius * 3.3], 1)); + selectedMaterial = graphPointMaterial(1.3, view.zoom); + scene.add(new THREE.Points(selectedGeometry, selectedMaterial)); + } + + runtimeRef.current = { + camera, + edgePosition, + edges: renderedEdges, + highlightedEdges: renderedHighlightedEdges, + highlightPosition, + nodeIds: layout.nodes.map(node => node.id), + nodePosition, + pointMaterials: selectedMaterial ? [nodeMaterial, selectedMaterial] : [nodeMaterial], + renderer, + scene, + selectedNodeId: selectedNode?.id, + selectedPosition, + }; + renderer.render(scene, camera); + return () => { + runtimeRef.current = undefined; + edgeGeometry.dispose(); + edgeMaterial.dispose(); + nodeGeometry.dispose(); + nodeMaterial.dispose(); + highlightGeometry?.dispose(); + highlightMaterial?.dispose(); + selectedGeometry?.dispose(); + selectedMaterial?.dispose(); + renderer.dispose(); + }; + }, [activeNodeIds, animatedNeighborhoodEdges, displayEdges, layout, props.graph.mode, props.selectedNodeId]); + + useEffect(() => { + const runtime = runtimeRef.current; + if (!runtime) return; + runtime.renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); + runtime.renderer.setSize(size.width, size.height, false); + for (const material of runtime.pointMaterials) { + const scale = material.uniforms.viewScale; + if (scale) scale.value = graphPointViewScale(view.zoom); + } + updateCamera(runtime.camera, view, size); + runtime.renderer.render(runtime.scene, runtime.camera); + }, [size, view]); + + useEffect(() => { + const currentNodes = baseLayout.nodes.map(node => { + const settledNode = layout.nodesById.get(node.id) ?? node; + const position = graphPosition(settledNode, livePositionsRef.current); + return {...node, x: position.x, y: position.y}; + }); + const labelSizes = new Map(); + for (const [nodeId, element] of labelRefs.current) { + labelSizes.set(nodeId, {height: element.offsetHeight, width: element.offsetWidth}); + } + const targets = graphFocusLayoutTargets( + currentNodes, + props.selectedNodeId, + animatedNeighborhoodEdges, + labelSizes, + Math.max(FOCUS_LAYOUT_ZOOM, viewRef.current.zoom), + ); + const simulationIds = new Set([...livePositionsRef.current.keys(), ...settledPositions.keys(), ...targets.keys()]); + const particles = [...simulationIds].flatMap(nodeId => { + const baseNode = baseLayout.nodesById.get(nodeId); + const currentNode = layout.nodesById.get(nodeId) ?? baseNode; + if (!baseNode || !currentNode) return []; + const start = livePositionsRef.current.get(nodeId) ?? currentNode; + const target = targets.get(nodeId) ?? baseNode; + return [ + { + id: nodeId, + targetX: target.x, + targetY: target.y, + velocityX: 0, + velocityY: 0, + x: start.x, + y: start.y, + }, + ]; + }); + const container = containerRef.current; + const reducedMotion = window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ?? false; + let frame = 0; + let lastFrame = performance.now(); + const startedAt = lastFrame; + + const settle = (): void => { + const resolvedPositions = new Map(); + for (const particle of particles) { + resolvedPositions.set(particle.id, {x: particle.targetX, y: particle.targetY}); + } + applyGraphPositions(runtimeRef.current, resolvedPositions, layout, size, viewRef.current, labelRefs.current); + const retainedTargets = new Map(); + for (const [nodeId, target] of targets) { + const baseNode = baseLayout.nodesById.get(nodeId); + if (baseNode && Math.hypot(target.x - baseNode.x, target.y - baseNode.y) > 0.01) { + retainedTargets.set(nodeId, target); + } + } + livePositionsRef.current = retainedTargets; + setSettledPositions(retainedTargets); + container?.removeAttribute('data-layout-animating'); + }; + + if ( + reducedMotion || + particles.every(particle => Math.hypot(particle.targetX - particle.x, particle.targetY - particle.y) < 0.01) + ) { + settle(); + return; + } + + container?.setAttribute('data-layout-animating', 'true'); + const animate = (now: number): void => { + const delta = Math.min(0.032, Math.max(0.001, (now - lastFrame) / 1000)); + lastFrame = now; + let movement = 0; + const positions = new Map(); + for (const particle of particles) { + const accelerationX = (particle.targetX - particle.x) * 108 - particle.velocityX * 13; + const accelerationY = (particle.targetY - particle.y) * 108 - particle.velocityY * 13; + particle.velocityX += accelerationX * delta; + particle.velocityY += accelerationY * delta; + particle.x += particle.velocityX * delta; + particle.y += particle.velocityY * delta; + movement = Math.max( + movement, + Math.hypot(particle.targetX - particle.x, particle.targetY - particle.y), + Math.hypot(particle.velocityX, particle.velocityY) * 0.035, + ); + positions.set(particle.id, {x: particle.x, y: particle.y}); + } + livePositionsRef.current = positions; + applyGraphPositions(runtimeRef.current, positions, layout, size, viewRef.current, labelRefs.current); + if (movement < 0.08 || now - startedAt >= 1250) { + settle(); + return; + } + frame = window.requestAnimationFrame(animate); + }; + frame = window.requestAnimationFrame(animate); + return () => { + window.cancelAnimationFrame(frame); + container?.removeAttribute('data-layout-animating'); + }; + }, [animatedNeighborhoodEdges, baseLayout, focusLayoutRevision, props.selectedNodeId]); + + const labels = useMemo( + () => + visibleLabels( + layout, + props.graph.mode, + size, + view, + props.selectedNodeId, + activeNodeIds, + highlightedNodeIds, + livePositionsRef.current, + ), + [activeNodeIds, highlightedNodeIds, layout, props.graph.mode, props.selectedNodeId, size, view], + ); + + const zoomAt = (factor: number, clientX = size.width / 2, clientY = size.height / 2): void => { + setView(current => zoomViewAt(current, factor, clientX, clientY, size)); + }; + + return ( +
+ { + const node = nearestNode( + layout, + view, + size, + event.nativeEvent.offsetX, + event.nativeEvent.offsetY, + livePositionsRef.current, + ); + if (node?.type === 'project') props.onOpenProject(node.projectId); + }} + onPointerDown={event => { + event.currentTarget.setPointerCapture(event.pointerId); + dragRef.current = {moved: false, pointerId: event.pointerId, x: event.clientX, y: event.clientY}; + }} + onPointerMove={event => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== event.pointerId) return; + const dx = event.clientX - drag.x; + const dy = event.clientY - drag.y; + if (Math.abs(dx) + Math.abs(dy) > 2) drag.moved = true; + drag.x = event.clientX; + drag.y = event.clientY; + setView(current => ({...current, x: current.x - dx / current.zoom, y: current.y + dy / current.zoom})); + }} + onPointerUp={event => { + const drag = dragRef.current; + if (drag && !drag.moved) { + const node = nearestNode( + layout, + view, + size, + event.nativeEvent.offsetX, + event.nativeEvent.offsetY, + livePositionsRef.current, + ); + props.onSelectNode(node?.id); + } + dragRef.current = undefined; + event.currentTarget.releasePointerCapture(event.pointerId); + }} + onWheel={event => { + event.preventDefault(); + zoomAt(graphWheelZoomFactor(event.deltaY), event.nativeEvent.offsetX, event.nativeEvent.offsetY); + }} + ref={canvasRef} + /> + + {renderError ? ( +
+

GPU rendering unavailable

+

{renderError}

+
+ ) : null} +
+ + + +
+
+ {Math.round(view.zoom * 100)}% + {view.zoom < 1.45 ? 'Zoom in to reveal symbols' : 'Detailed labels visible'} +
+
+ ); +} +function buildGraphLayout( + graph: GraphVisualization, + sizeMetric: GraphSizeMetric, + sizingEdges: readonly GraphEdge[], +): GraphLayout { + const sizeValues = graphNodeSizeValues(sizingEdges, sizeMetric); + const nodes = graph.mode === 'overview' ? overviewLayout(graph.nodes) : detailLayout(graph.nodes, sizeValues); + const nodesById = new Map(nodes.map(node => [node.id, node])); + const extentX = Math.max(260, ...nodes.map(node => Math.abs(node.x) + node.radius)); + const extentY = Math.max(200, ...nodes.map(node => Math.abs(node.y) + node.radius)); + return {bounds: {height: extentY * 2.2, width: extentX * 2.2}, nodes, nodesById}; +} + +function graphLayoutWithPositions(layout: GraphLayout, positions: ReadonlyMap): GraphLayout { + if (positions.size === 0) return layout; + const nodes = layout.nodes.map(node => { + const position = positions.get(node.id); + return position ? {...node, x: position.x, y: position.y} : node; + }); + const nodesById = new Map(nodes.map(node => [node.id, node])); + const extentX = Math.max(260, ...nodes.map(node => Math.abs(node.x) + node.radius)); + const extentY = Math.max(200, ...nodes.map(node => Math.abs(node.y) + node.radius)); + return {bounds: {height: extentY * 2.2, width: extentX * 2.2}, nodes, nodesById}; +} + +export function graphFocusLayoutTargets( + nodes: readonly { + readonly id: string; + readonly label: string; + readonly radius: number; + readonly x: number; + readonly y: number; + }[], + selectedNodeId: string | undefined, + edges: readonly Pick[], + labelSizes: ReadonlyMap = new Map(), + zoom = FOCUS_LAYOUT_ZOOM, +): ReadonlyMap { + if (!selectedNodeId) return new Map(); + const nodesById = new Map(nodes.map(node => [node.id, node])); + const selectedNode = nodesById.get(selectedNodeId); + if (!selectedNode) return new Map(); + const neighborIds = new Set(); + for (const edge of edges) { + if (edge.sourceId === selectedNodeId && nodesById.has(edge.targetId)) neighborIds.add(edge.targetId); + if (edge.targetId === selectedNodeId && nodesById.has(edge.sourceId)) neighborIds.add(edge.sourceId); + } + neighborIds.delete(selectedNodeId); + const orderedNeighbors = [...neighborIds] + .map(nodeId => nodesById.get(nodeId)) + .filter(node => node !== undefined) + .sort((left, right) => compareCodeUnits(left.label, right.label) || compareCodeUnits(left.id, right.id)); + const highlightedIds = new Set([selectedNodeId, ...neighborIds]); + const visibleObstacles = nodes + .filter(node => !highlightedIds.has(node.id) && labelSizes.has(node.id)) + .sort((left, right) => compareCodeUnits(left.id, right.id)) + .slice(0, 180); + const focusNodes = [ + { + anchorX: selectedNode.x, + anchorY: selectedNode.y, + fixed: true, + highlighted: true, + ...selectedNode, + }, + ...orderedNeighbors.map(node => ({ + anchorX: node.x, + anchorY: node.y, + fixed: false, + highlighted: true, + ...node, + })), + ...visibleObstacles.map(node => ({ + anchorX: node.x, + anchorY: node.y, + fixed: false, + highlighted: false, + ...node, + })), + ]; + const safeZoom = Math.max(0.5, zoom); + const animatedNeighbors = focusNodes.filter(node => node.highlighted && !node.fixed); + const maximumLabelWidth = Math.max( + 72, + ...animatedNeighbors.map(node => labelSizes.get(node.id)?.width ?? Math.min(150, node.label.length * 6.2)), + ); + const maximumLabelHeight = Math.max(14, ...animatedNeighbors.map(node => labelSizes.get(node.id)?.height ?? 14)); + const columns = Math.max(2, Math.ceil(Math.sqrt((animatedNeighbors.length + 1) * 0.35))); + const rows = Math.ceil((animatedNeighbors.length + 1) / columns); + const cellWidth = (Math.min(150, maximumLabelWidth) + 14) / safeZoom; + const cellHeight = (maximumLabelHeight + 10) / safeZoom; + const slots = Array.from({length: rows * columns}, (_, index) => { + const column = index % columns; + const row = Math.floor(index / columns); + return { + x: (column - (columns - 1) / 2) * cellWidth, + y: ((rows - 1) / 2 - row) * cellHeight, + }; + }); + const centerSlot = slots.reduce( + (closest, slot, index) => + Math.hypot(slot.x, slot.y) < closest.distance ? {distance: Math.hypot(slot.x, slot.y), index} : closest, + {distance: Number.POSITIVE_INFINITY, index: 0}, + ); + slots.splice(centerSlot.index, 1); + slots.sort( + (left, right) => Math.hypot(left.x, left.y) - Math.hypot(right.x, right.y) || left.y - right.y || left.x - right.x, + ); + for (const [index, node] of animatedNeighbors.entries()) { + const slot = slots[index] ?? {x: 0, y: 0}; + node.x = selectedNode.x + slot.x; + node.y = selectedNode.y + slot.y; + node.anchorX = node.x; + node.anchorY = node.y; + let deltaX = slot.x; + let deltaY = slot.y; + let distance = Math.hypot(deltaX, deltaY); + const minimumDistance = (selectedNode.radius * 1.25 + node.radius * 1.25 + 22) / safeZoom; + if (distance < 0.001) { + const angle = (Math.abs(hashString(node.id)) % 6283) / 1000 + index * 2.399963; + deltaX = Math.cos(angle); + deltaY = Math.sin(angle); + distance = 1; + } + if (distance < minimumDistance) { + node.x = selectedNode.x + (deltaX / distance) * minimumDistance; + node.y = selectedNode.y + (deltaY / distance) * minimumDistance; + } + } + + // Preserve the full relaxation pass for ordinary neighborhoods while bounding + // maximum-cardinality focus work. Dense graphs benefit more from responsive + // interaction than from repeatedly refining already-overlapping offscreen labels. + const collisionIterations = Math.max(10, Math.min(18, Math.floor(5_000 / focusNodes.length))); + const movableFocusNodes = focusNodes.filter(node => !node.fixed); + for (let iteration = 0; iteration < collisionIterations; iteration += 1) { + for (const node of movableFocusNodes) { + node.x += (node.anchorX - node.x) * 0.006; + node.y += (node.anchorY - node.y) * 0.006; + } + for (const [leftIndex, rightIndex] of focusCollisionPairs(focusNodes, labelSizes, safeZoom)) { + separateFocusNodes(focusNodes[leftIndex]!, focusNodes[rightIndex]!, labelSizes, safeZoom); + } + for (const node of animatedNeighbors) { + const deltaX = node.x - selectedNode.x; + const deltaY = node.y - selectedNode.y; + const distance = Math.max(0.001, Math.hypot(deltaX, deltaY)); + const minimumDistance = (selectedNode.radius * 1.25 + node.radius * 1.25 + 22) / safeZoom; + if (distance < minimumDistance) { + node.x = selectedNode.x + (deltaX / distance) * minimumDistance; + node.y = selectedNode.y + (deltaY / distance) * minimumDistance; + } + } + } + return new Map(focusNodes.map(node => [node.id, {x: node.x, y: node.y}])); +} + +function focusCollisionPairs( + nodes: readonly { + readonly fixed: boolean; + readonly highlighted: boolean; + readonly id: string; + readonly label: string; + readonly radius: number; + readonly x: number; + readonly y: number; + }[], + labelSizes: ReadonlyMap, + zoom: number, +): readonly (readonly [number, number])[] { + const bounds = nodes + .map((node, index) => { + const boxes = focusNodeBoxes(node, labelSizes.get(node.id), zoom, node.fixed); + return { + bottom: Math.max(...boxes.map(box => box.bottom)), + highlighted: node.highlighted, + index, + left: Math.min(...boxes.map(box => box.left)), + right: Math.max(...boxes.map(box => box.right)), + top: Math.min(...boxes.map(box => box.top)), + }; + }) + .sort((left, right) => left.left - right.left || left.index - right.index); + const pairs: Array = []; + for (const [leftPosition, left] of bounds.entries()) { + for (let rightPosition = leftPosition + 1; rightPosition < bounds.length; rightPosition += 1) { + const right = bounds[rightPosition]!; + if (right.left >= left.right) break; + if (!left.highlighted && !right.highlighted) continue; + if (Math.min(left.bottom, right.bottom) <= Math.max(left.top, right.top)) continue; + pairs.push([left.index, right.index]); + } + } + return pairs; +} + +function separateFocusNodes( + left: { + readonly fixed: boolean; + readonly id: string; + readonly label: string; + readonly radius: number; + x: number; + y: number; + }, + right: { + readonly fixed: boolean; + readonly id: string; + readonly label: string; + readonly radius: number; + x: number; + y: number; + }, + labelSizes: ReadonlyMap, + zoom: number, +): void { + const leftBoxes = focusNodeBoxes(left, labelSizes.get(left.id), zoom, left.fixed); + const rightBoxes = focusNodeBoxes(right, labelSizes.get(right.id), zoom, right.fixed); + for (const leftBox of leftBoxes) { + for (const rightBox of rightBoxes) { + const overlapX = Math.min(leftBox.right, rightBox.right) - Math.max(leftBox.left, rightBox.left); + const overlapY = Math.min(leftBox.bottom, rightBox.bottom) - Math.max(leftBox.top, rightBox.top); + if (overlapX <= 0 || overlapY <= 0) continue; + const leftCenterX = (leftBox.left + leftBox.right) / 2; + const leftCenterY = (leftBox.top + leftBox.bottom) / 2; + const rightCenterX = (rightBox.left + rightBox.right) / 2; + const rightCenterY = (rightBox.top + rightBox.bottom) / 2; + const fallback = hashString(`${left.id}:${right.id}`); + if (overlapX < overlapY) { + const direction = + leftCenterX === rightCenterX ? (fallback % 2 === 0 ? -1 : 1) : Math.sign(leftCenterX - rightCenterX); + moveFocusPair(left, right, direction * (overlapX + 2 / zoom), 0); + } else { + const direction = + leftCenterY === rightCenterY ? (fallback % 2 === 0 ? -1 : 1) : Math.sign(leftCenterY - rightCenterY); + moveFocusPair(left, right, 0, direction * (overlapY + 2 / zoom)); + } + } + } +} + +function moveFocusPair( + left: {readonly fixed: boolean; x: number; y: number}, + right: {readonly fixed: boolean; x: number; y: number}, + deltaX: number, + deltaY: number, +): void { + if (left.fixed && right.fixed) return; + if (left.fixed) { + right.x -= deltaX; + right.y -= deltaY; + return; + } + if (right.fixed) { + left.x += deltaX; + left.y += deltaY; + return; + } + left.x += deltaX / 2; + left.y += deltaY / 2; + right.x -= deltaX / 2; + right.y -= deltaY / 2; +} + +function focusNodeBoxes( + node: {readonly label: string; readonly radius: number; readonly x: number; readonly y: number}, + measured: {readonly height: number; readonly width: number} | undefined, + zoom: number, + selected: boolean, +): readonly {readonly bottom: number; readonly left: number; readonly right: number; readonly top: number}[] { + const nodeHalfSize = (node.radius * 1.25 + 4) / zoom; + const estimatedWidth = Math.min(selected ? 300 : 220, Math.max(28, node.label.length * 6.2 + (selected ? 14 : 0))); + const labelWidth = (measured?.width ?? estimatedWidth) / zoom; + const labelHeight = (measured?.height ?? (selected ? 22 : 14)) / zoom; + const labelLeft = node.x + (node.radius + 4) / zoom; + const margin = 3 / zoom; + const nodeBox = { + bottom: node.y + nodeHalfSize + margin, + left: node.x - nodeHalfSize - margin, + right: node.x + nodeHalfSize + margin, + top: node.y - nodeHalfSize - margin, + }; + if (!measured && !selected) return [nodeBox]; + return [ + nodeBox, + { + bottom: node.y + labelHeight / 2 + margin, + left: labelLeft - margin, + right: labelLeft + labelWidth + margin, + top: node.y - labelHeight / 2 - margin, + }, + ]; +} + +function overviewLayout(nodes: readonly GraphNode[]): readonly PositionedNode[] { + const ordered = [...nodes].sort( + (left, right) => + (right.symbolCount ?? right.degree) - (left.symbolCount ?? left.degree) || + compareCodeUnits(left.label, right.label), + ); + return ordered.map((node, index) => { + const angle = index * 2.399963; + const ring = index === 0 ? 0 : 78 + Math.sqrt(index) * 84; + return positionNode(node, Math.cos(angle) * ring, Math.sin(angle) * ring, index); + }); +} + +function detailLayout(nodes: readonly GraphNode[], sizeValues: ReadonlyMap): readonly PositionedNode[] { + const groups = new Map(); + for (const node of nodes) { + const group = graphGroup(node); + const items = groups.get(group) ?? []; + items.push(node); + groups.set(group, items); + } + const orderedGroups = [...groups].sort( + ([leftName, left], [rightName, right]) => right.length - left.length || compareCodeUnits(leftName, rightName), + ); + const output: PositionedNode[] = []; + for (const [groupIndex, [, items]] of orderedGroups.entries()) { + const groupAngle = groupIndex * 2.399963; + const groupRadius = orderedGroups.length === 1 ? 0 : 120 + Math.sqrt(groupIndex) * 135; + const centerX = Math.cos(groupAngle) * groupRadius; + const centerY = Math.sin(groupAngle) * groupRadius; + const ordered = [...items].sort( + (left, right) => right.degree - left.degree || compareCodeUnits(left.label, right.label), + ); + for (const [itemIndex, node] of ordered.entries()) { + const angle = itemIndex * 2.399963 + groupAngle; + const radius = itemIndex === 0 ? 0 : 17 * Math.sqrt(itemIndex); + output.push( + positionNode( + node, + centerX + Math.cos(angle) * radius, + centerY + Math.sin(angle) * radius, + groupIndex, + sizeValues.get(node.id) ?? 0, + ), + ); + } + } + return output; +} + +function positionNode( + node: GraphNode, + x: number, + y: number, + colorIndex: number, + sizeValue = node.degree, +): PositionedNode { + const radius = + node.type === 'project' + ? 8 + Math.min(14, Math.sqrt(Math.max(1, Math.log2((node.symbolCount ?? sizeValue) + 1))) * 3) + : 4 + Math.min(11, Math.log2(Math.max(0, sizeValue) + 1) * 2); + return { + ...node, + color: new THREE.Color(colorForNode(node, colorIndex)), + radius, + x, + y, + }; +} + +function colorForNode(node: GraphNode, fallbackIndex: number): string { + if (node.type === 'project') return GRAPH_PALETTE[fallbackIndex % GRAPH_PALETTE.length]!; + const key = node.projectId || node.kind; + return GRAPH_PALETTE[Math.abs(hashString(key)) % GRAPH_PALETTE.length]!; +} + +function graphGroup(node: GraphNode): string { + if (!node.path) return node.projectId; + const parts = node.path.split('/'); + return parts.slice(0, Math.min(2, Math.max(1, parts.length - 1))).join('/'); +} + +function fittedView(layout: GraphLayout, size: {readonly height: number; readonly width: number}): ViewState { + const padding = 1.12; + const zoom = Math.min( + 1.6, + Math.max( + MIN_ZOOM, + Math.min(size.width / (layout.bounds.width * padding), size.height / (layout.bounds.height * padding)), + ), + ); + return {x: 0, y: 0, zoom: Number.isFinite(zoom) ? zoom : 1}; +} + +export function graphFocusTarget( + current: ViewState, + node: {readonly x: number; readonly y: number}, + mode: GraphVisualization['mode'], +): ViewState { + const targetZoom = SEARCH_FOCUS_ZOOM[mode]; + const currentZoom = Number.isFinite(current.zoom) ? current.zoom : targetZoom; + return { + x: Number.isFinite(node.x) ? node.x : Number.isFinite(current.x) ? current.x : 0, + y: Number.isFinite(node.y) ? node.y : Number.isFinite(current.y) ? current.y : 0, + zoom: Math.min(targetZoom * 1.35, Math.max(currentZoom, targetZoom)), + }; +} + +export function graphWheelZoomFactor(deltaY: number): number { + if (Number.isNaN(deltaY)) return 1; + return Math.max(0.72, Math.min(1.38, Math.exp(-deltaY * 0.0012))); +} + +function updateCamera( + camera: THREE.OrthographicCamera, + view: ViewState, + size: {readonly height: number; readonly width: number}, +): void { + camera.left = -size.width / 2 / view.zoom; + camera.right = size.width / 2 / view.zoom; + camera.top = size.height / 2 / view.zoom; + camera.bottom = -size.height / 2 / view.zoom; + camera.near = 0.1; + camera.far = 200; + camera.position.set(view.x, view.y, 100); + camera.updateProjectionMatrix(); +} + +function graphPosition( + node: {readonly id: string; readonly x: number; readonly y: number}, + positions?: ReadonlyMap, +): GraphPosition { + return positions?.get(node.id) ?? node; +} + +function applyGraphPositions( + runtime: GraphRuntime | undefined, + positions: ReadonlyMap, + layout: GraphLayout, + size: {readonly height: number; readonly width: number}, + view: ViewState, + labelElements: ReadonlyMap, +): void { + if (runtime) { + for (const [index, nodeId] of runtime.nodeIds.entries()) { + const node = layout.nodesById.get(nodeId); + if (!node) continue; + const position = graphPosition(node, positions); + runtime.nodePosition.setXYZ(index, position.x, position.y, 1); + } + runtime.nodePosition.needsUpdate = true; + for (const [index, edge] of runtime.edges.entries()) { + const source = layout.nodesById.get(edge.sourceId); + const target = layout.nodesById.get(edge.targetId); + if (!source || !target) continue; + const sourcePosition = graphPosition(source, positions); + const targetPosition = graphPosition(target, positions); + runtime.edgePosition.setXYZ(index * 2, sourcePosition.x, sourcePosition.y, 0); + runtime.edgePosition.setXYZ(index * 2 + 1, targetPosition.x, targetPosition.y, 0); + } + runtime.edgePosition.needsUpdate = true; + if (runtime.highlightPosition) { + const highlightPositions = directionalEdgePositions(runtime.highlightedEdges, layout.nodesById, positions); + if (highlightPositions.length === runtime.highlightPosition.array.length) { + runtime.highlightPosition.array.set(highlightPositions); + runtime.highlightPosition.needsUpdate = true; + } + } + if (runtime.selectedNodeId && runtime.selectedPosition) { + const selectedNode = layout.nodesById.get(runtime.selectedNodeId); + if (selectedNode) { + const selectedPosition = graphPosition(selectedNode, positions); + runtime.selectedPosition.setXYZ(0, selectedPosition.x, selectedPosition.y, 2); + runtime.selectedPosition.needsUpdate = true; + } + } + } + + for (const [nodeId, element] of labelElements) { + const node = layout.nodesById.get(nodeId); + if (!node) continue; + const position = graphPosition(node, positions); + const x = size.width / 2 + (position.x - view.x) * view.zoom; + const y = size.height / 2 - (position.y - view.y) * view.zoom; + element.style.left = `${x + node.radius + 4}px`; + element.style.top = `${y}px`; + } + if (runtime) runtime.renderer.render(runtime.scene, runtime.camera); +} + +function zoomViewAt( + view: ViewState, + factor: number, + screenX: number, + screenY: number, + size: {readonly height: number; readonly width: number}, +): ViewState { + const zoom = Math.max(MIN_ZOOM, Math.min(MAX_ZOOM, view.zoom * factor)); + const dx = screenX - size.width / 2; + const dy = screenY - size.height / 2; + const worldX = view.x + dx / view.zoom; + const worldY = view.y - dy / view.zoom; + return {x: worldX - dx / zoom, y: worldY + dy / zoom, zoom}; +} + +function nearestNode( + layout: GraphLayout, + view: ViewState, + size: {readonly height: number; readonly width: number}, + screenX: number, + screenY: number, + positions?: ReadonlyMap, +): PositionedNode | undefined { + const worldX = view.x + (screenX - size.width / 2) / view.zoom; + const worldY = view.y - (screenY - size.height / 2) / view.zoom; + let selected: PositionedNode | undefined; + let selectedDistance = Number.POSITIVE_INFINITY; + for (const node of layout.nodes) { + const position = graphPosition(node, positions); + const distance = Math.hypot(position.x - worldX, position.y - worldY); + const hitRadius = Math.max(node.radius * 1.45, 10 / view.zoom); + if (distance <= hitRadius && distance < selectedDistance) { + selected = node; + selectedDistance = distance; + } + } + return selected; +} + +function visibleLabels( + layout: GraphLayout, + mode: GraphVisualization['mode'], + size: {readonly height: number; readonly width: number}, + view: ViewState, + selectedNodeId?: string, + activeNodeIds?: ReadonlySet, + highlightedNodeIds?: ReadonlySet, + positions?: ReadonlyMap, +): readonly {readonly node: PositionedNode; readonly x: number; readonly y: number}[] { + const baseMaximum = + mode === 'overview' + ? view.zoom < 0.65 + ? 18 + : 80 + : view.zoom < 0.75 + ? 8 + : view.zoom < 1.45 + ? 24 + : view.zoom < 3 + ? 72 + : 180; + const highlightedMaximum = + view.zoom < 0.75 + ? 0 + : view.zoom < 1.45 + ? Math.min(24, highlightedNodeIds?.size ?? 0) + : Math.min(MAX_FOCUSED_LABELS + 1, highlightedNodeIds?.size ?? 0); + const maximum = Math.max(baseMaximum, highlightedMaximum); + let focusedLabelCount = 0; + return [...layout.nodes] + .filter(node => !activeNodeIds || activeNodeIds.has(node.id)) + .flatMap(node => { + const position = graphPosition(node, positions); + const x = size.width / 2 + (position.x - view.x) * view.zoom; + const y = size.height / 2 - (position.y - view.y) * view.zoom; + return x < -80 || x > size.width + 80 || y < -30 || y > size.height + 30 ? [] : [{node, x, y}]; + }) + .sort((left, right) => { + if (left.node.id === selectedNodeId) return -1; + if (right.node.id === selectedNodeId) return 1; + if (highlightedNodeIds?.has(left.node.id) && !highlightedNodeIds.has(right.node.id)) return -1; + if (highlightedNodeIds?.has(right.node.id) && !highlightedNodeIds.has(left.node.id)) return 1; + return ( + right.node.degree - left.node.degree || + right.node.radius - left.node.radius || + compareCodeUnits(left.node.label, right.node.label) + ); + }) + .filter(({node}) => { + if (node.id === selectedNodeId || !highlightedNodeIds?.has(node.id)) return true; + focusedLabelCount += 1; + return focusedLabelCount <= MAX_FOCUSED_LABELS; + }) + .map(({node, x, y}) => ({node, x: x + node.radius + 4, y})) + .slice(0, maximum); +} + +function directionalEdgePositions( + edges: readonly GraphEdge[], + nodesById: ReadonlyMap, + positionOverrides?: ReadonlyMap, +): readonly number[] { + const positions: number[] = []; + for (const edge of edges.slice(0, MAX_ANIMATED_NEIGHBOR_EDGES)) { + const source = nodesById.get(edge.sourceId); + const target = nodesById.get(edge.targetId); + if (!source || !target) continue; + const sourcePosition = graphPosition(source, positionOverrides); + const targetPosition = graphPosition(target, positionOverrides); + let dx = targetPosition.x - sourcePosition.x; + let dy = targetPosition.y - sourcePosition.y; + let length = Math.hypot(dx, dy); + if (length < 0.001) { + const angle = (Math.abs(hashString(edge.id)) % 6283) / 1000; + dx = Math.cos(angle) * 0.001; + dy = Math.sin(angle) * 0.001; + length = 0.001; + } + const unitX = dx / length; + const unitY = dy / length; + const tipX = targetPosition.x - unitX * (target.radius + 2); + const tipY = targetPosition.y - unitY * (target.radius + 2); + const arrowLength = Math.min(8, Math.max(4, length * 0.16)); + const wingX = tipX - unitX * arrowLength; + const wingY = tipY - unitY * arrowLength; + const normalX = -unitY * arrowLength * 0.55; + const normalY = unitX * arrowLength * 0.55; + positions.push( + sourcePosition.x, + sourcePosition.y, + 1.5, + tipX, + tipY, + 1.5, + tipX, + tipY, + 1.5, + wingX + normalX, + wingY + normalY, + 1.5, + tipX, + tipY, + 1.5, + wingX - normalX, + wingY - normalY, + 1.5, + ); + } + return positions; +} + +function graphPointMaterial(scale: number, zoom: number): THREE.ShaderMaterial { + return new THREE.ShaderMaterial({ + blending: THREE.AdditiveBlending, + depthWrite: false, + fragmentShader: ` + varying vec3 vColor; + void main() { + vec2 point = gl_PointCoord - vec2(0.5); + float distanceToCenter = length(point); + if (distanceToCenter > 0.5) discard; + float glow = smoothstep(0.5, 0.05, distanceToCenter); + float core = smoothstep(0.24, 0.05, distanceToCenter); + gl_FragColor = vec4(vColor + core * 0.32, glow * 0.94); + } + `, + transparent: true, + uniforms: { + viewScale: {value: graphPointViewScale(zoom)}, + }, + vertexColors: true, + vertexShader: ` + attribute float pointSize; + uniform float viewScale; + varying vec3 vColor; + void main() { + vColor = color; + gl_PointSize = max(3.0, pointSize * ${scale.toFixed(2)} * viewScale); + gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); + } + `, + }); +} + +function graphPointViewScale(zoom: number): number { + return Math.min(1.25, Math.max(0.32, zoom * 0.75)); +} + +function hashString(value: string): number { + let hash = 2166136261; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return hash | 0; +} + +function lerp(start: number, end: number, progress: number): number { + return start + (end - start) * progress; +} +export function managerGraphClientRenderProxy( + graph: GraphVisualization, + size: {readonly height: number; readonly width: number} = {height: 720, width: 1_280}, +): {readonly labels: number; readonly matchedEdges: number; readonly nodes: number} { + const layout = buildGraphLayout(graph, 'connections', graph.edges); + const view = fittedView(layout, size); + let matchedEdges = 0; + for (const edge of graph.edges) { + if (layout.nodesById.has(edge.sourceId) && layout.nodesById.has(edge.targetId)) matchedEdges += 1; + } + return { + labels: visibleLabels(layout, graph.mode, size, view).length, + matchedEdges, + nodes: layout.nodes.length, + }; +} diff --git a/src/manager_graph_workspace.tsx b/src/manager_graph_workspace.tsx new file mode 100644 index 00000000..088ed736 --- /dev/null +++ b/src/manager_graph_workspace.tsx @@ -0,0 +1,1144 @@ +import React, {useEffect, useMemo, useRef, useState} from 'react'; +import type {CodeGraphLocalDiagnosticsReport} from './code_graph/diagnostics.js'; +import {compareCodeUnits} from './code_graph/ordering.js'; +import {type ManagerGraphVisualizationLimits} from './manager_graph_limits.js'; +import { + cacheGraphNodeDetail, + compactNumber, + createGraphQueryRequestGate, + DEFAULT_QUERY_WORKING_SET, + DEFAULT_WORKING_SET, + GRAPH_QUERY_DEBOUNCE_MILLISECONDS, + GRAPH_QUERY_MAXIMUM_LENGTH, + graphAnalysisRequestIsCurrent, + graphBuildShouldDisplay, + graphCatalogContinuationHasMore, + graphCatalogPageOffsets, + graphCatalogSearchOptions, + graphLocalAssociationText, + graphNodeDetailRequestIsCurrent, + graphOverviewSizeLabel, + graphProjectBadge, + graphRepositoryOptionLabel, + graphRequestIsCurrent, + graphWithNodeNeighborhood, + isAbortError, + managerGraphDebouncedQueryCandidate, + managerGraphQueryCandidate, + MAX_QUERY_WORKING_SET, + MAX_WORKING_SET, + mergeGraphRepositoryGroups, + relationLabel, + resolveGraphSelection, + type GraphAdministrationAction, + type GraphAnalysis, + type GraphCatalog, + type GraphCatalogContinuation, + type GraphCatalogPage, + type GraphCatalogSearchOptions, + type GraphFocusMode, + type GraphNodeDetail, + type GraphQueryVisualization, + type GraphRepositoryGroup, + type GraphSizeMetric, + type GraphViewPage, + type GraphVisualization, +} from './manager_graph_model.js'; +import { + GraphAdministration, + GraphAutomaticCompactionProgress, + GraphBuildProgress, + GraphEmptyState, + GraphMaintenanceProgress, + GraphSummary, + NodeInspector, +} from './manager_graph_panels.js'; +import {ThreeGraph} from './manager_graph_scene.js'; + +export function GraphWorkspace(props: { + readonly administration?: CodeGraphLocalDiagnosticsReport; + readonly administrationBusy?: string; + readonly administrationOutput?: string; + readonly catalog?: GraphCatalog; + readonly catalogError?: string; + readonly loadAnalysis: (repositoryId: string, snapshotId: string, signal: AbortSignal) => Promise; + readonly loadGraph: ( + repositoryId: string, + snapshotId: string, + projectId: string, + limits: ManagerGraphVisualizationLimits, + signal: AbortSignal, + ) => Promise; + readonly loadCatalogPage: ( + repositoryId: string, + snapshotId: string, + projectOffset: number, + workspaceOffset: number, + query: string, + signal: AbortSignal, + ) => Promise; + readonly loadNodeDetail: ( + repositoryId: string, + snapshotId: string, + nodeId: string, + signal: AbortSignal, + ) => Promise; + readonly loadQuery: ( + repositoryId: string, + snapshotId: string, + query: string, + limits: ManagerGraphVisualizationLimits, + signal: AbortSignal, + ) => Promise; + readonly loadViewsPage: ( + repositoryId: string, + offset: number, + query: string, + signal: AbortSignal, + ) => Promise; + readonly onAdministrationAction?: (action: GraphAdministrationAction) => void; + readonly onDiagnostics?: (options: {readonly analyze: boolean; readonly deep: boolean}) => void; + readonly onRefresh: () => void; +}): React.ReactElement { + const [repositoryId, setRepositoryId] = useState(''); + const [viewId, setViewId] = useState(''); + const [projectId, setProjectId] = useState('all'); + const [baseGraph, setBaseGraph] = useState(); + const [workingSet, setWorkingSet] = useState(DEFAULT_WORKING_SET); + const [expandedNeighborhood, setExpandedNeighborhood] = useState(); + const [selectedNodeId, setSelectedNodeId] = useState(); + const [focusRequest, setFocusRequest] = useState<{readonly nodeId: string; readonly sequence: number} | undefined>(); + const focusSequence = useRef(0); + const [search, setSearch] = useState(''); + const [queryInput, setQueryInput] = useState(''); + const [activeQuery, setActiveQuery] = useState(''); + const [queryGraph, setQueryGraph] = useState(); + const [queryLoading, setQueryLoading] = useState(false); + const [queryError, setQueryError] = useState(''); + const [queryAttempt, setQueryAttempt] = useState(0); + const [queryWorkingSet, setQueryWorkingSet] = useState(DEFAULT_QUERY_WORKING_SET); + const queryRequestGate = useRef(createGraphQueryRequestGate()); + const [relationFilter, setRelationFilter] = useState('all'); + const [focusMode, setFocusMode] = useState('all'); + const [sizeMetric, setSizeMetric] = useState('connections'); + const [nodeDetail, setNodeDetail] = useState(); + const [nodeDetailLoading, setNodeDetailLoading] = useState(false); + const [nodeDetailError, setNodeDetailError] = useState(''); + const nodeDetailCache = useRef(new Map()); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + const [analysis, setAnalysis] = useState(); + const [analysisLoading, setAnalysisLoading] = useState(false); + const [analysisError, setAnalysisError] = useState(''); + const analysisRequestSequence = useRef(0); + const analysisAbortController = useRef(undefined); + const graphRequestSequence = useRef(0); + const [catalogAdditions, setCatalogAdditions] = useState([]); + const [catalogQuery, setCatalogQuery] = useState(''); + const [catalogLoading, setCatalogLoading] = useState(false); + const [catalogError, setCatalogError] = useState(''); + const [catalogSearchResult, setCatalogSearchResult] = useState< + {readonly options: GraphCatalogSearchOptions; readonly query: string} | undefined + >(); + const [catalogContinuation, setCatalogContinuation] = useState(); + const catalogAbortController = useRef(undefined); + const catalogRequestSequence = useRef(0); + const baseCatalogIdentity = useMemo( + () => + (props.catalog?.repositories ?? []) + .flatMap(group => group.views.map(view => `${view.id}:${view.snapshot.id}`)) + .sort(compareCodeUnits) + .join('|'), + [props.catalog?.repositories], + ); + const repositories = useMemo( + () => mergeGraphRepositoryGroups(props.catalog?.repositories ?? [], catalogAdditions), + [catalogAdditions, props.catalog?.repositories], + ); + const repositoryGroup = repositories.find(candidate => candidate.id === repositoryId) ?? repositories[0]; + const repository = + repositoryGroup?.views.find(candidate => candidate.id === viewId) ?? + repositoryGroup?.views.find(candidate => candidate.id === repositoryGroup.defaultViewId) ?? + repositoryGroup?.views[0]; + const baseRepositoryGroup = (props.catalog?.repositories ?? []).find( + candidate => candidate.id === repositoryGroup?.id, + ); + const baseRepository = baseRepositoryGroup?.views.find(candidate => candidate.id === repository?.id); + const analysisScope = `${repository?.id ?? ''}:${repository?.snapshot.id ?? ''}`; + const analysisScopeRef = useRef(analysisScope); + analysisScopeRef.current = analysisScope; + const graphScope = `${analysisScope}:${projectId}:${workingSet.nodeLimit}:${workingSet.edgeLimit}`; + const graphScopeRef = useRef(graphScope); + graphScopeRef.current = graphScope; + const queryScope = `${analysisScope}:${activeQuery}:${queryAttempt}:${queryWorkingSet.nodeLimit}:${queryWorkingSet.edgeLimit}`; + const graphSource = activeQuery ? queryGraph : baseGraph; + const graph = useMemo( + () => + graphSource && expandedNeighborhood ? graphWithNodeNeighborhood(graphSource, expandedNeighborhood) : graphSource, + [expandedNeighborhood, graphSource], + ); + const selectedNode = graph?.nodes.find(node => node.id === selectedNodeId); + const relations = useMemo( + () => [...new Set(graph?.edges.map(edge => edge.relation) ?? [])].sort(compareCodeUnits), + [graph], + ); + const activeBuilds = (props.catalog?.builds ?? []).filter(graphBuildShouldDisplay); + const selectedRepositoryIsIndexing = activeBuilds.some( + build => + repository !== undefined && + build.identity.checkoutId === repository.checkoutId && + build.identity.worktreeId === repository.worktreeId && + (build.state === 'queued' || build.state === 'running'), + ); + const workingSetAtMaximum = activeQuery + ? queryWorkingSet.nodeLimit >= MAX_QUERY_WORKING_SET.nodeLimit && + queryWorkingSet.edgeLimit >= MAX_QUERY_WORKING_SET.edgeLimit + : workingSet.nodeLimit >= MAX_WORKING_SET.nodeLimit && workingSet.edgeLimit >= MAX_WORKING_SET.edgeLimit; + const projectCatalogHasMore = graphCatalogContinuationHasMore( + catalogContinuation, + repository?.id, + 'projectHasMore', + repository?.projectsTruncated ?? false, + ); + const workspaceCatalogHasMore = graphCatalogContinuationHasMore( + catalogContinuation, + repository?.id, + 'workspaceHasMore', + repository?.workspacesTruncated ?? false, + ); + const viewCatalogHasMore = graphCatalogContinuationHasMore( + catalogContinuation, + repository?.id, + 'viewHasMore', + repositoryGroup?.viewsTruncated ?? false, + ); + + useEffect(() => { + const selection = resolveGraphSelection(repositories, repositoryId, viewId); + if (selection.repositoryId !== repositoryId) { + setRepositoryId(selection.repositoryId); + setProjectId('all'); + setWorkingSet(DEFAULT_WORKING_SET); + } + if (selection.viewId !== viewId) { + setViewId(selection.viewId); + setProjectId('all'); + setWorkingSet(DEFAULT_WORKING_SET); + } + }, [repositories, repositoryId, viewId]); + + useEffect(() => { + if (!repository) { + setBaseGraph(undefined); + setExpandedNeighborhood(undefined); + return; + } + const requestSequence = graphRequestSequence.current + 1; + graphRequestSequence.current = requestSequence; + const requestedScope = graphScope; + const controller = new AbortController(); + setLoading(true); + setError(''); + setSelectedNodeId(undefined); + setExpandedNeighborhood(undefined); + setFocusRequest(undefined); + setFocusMode('all'); + setRelationFilter('all'); + setSizeMetric('connections'); + void props + .loadGraph(repository.id, repository.snapshot.id, projectId, workingSet, controller.signal) + .then(next => { + if ( + graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) + ) { + setBaseGraph(next); + } + }) + .catch(cause => { + if ( + !isAbortError(cause) && + graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) + ) { + setBaseGraph(undefined); + setError(cause instanceof Error ? cause.message : String(cause)); + } + }) + .finally(() => { + if ( + graphRequestIsCurrent(graphRequestSequence.current, requestSequence, graphScopeRef.current, requestedScope) + ) { + setLoading(false); + } + }); + return () => { + controller.abort(); + graphRequestSequence.current += 1; + }; + }, [graphScope, projectId, props.loadGraph, repository?.id, repository?.snapshot.id, workingSet]); + + useEffect(() => { + const candidate = managerGraphDebouncedQueryCandidate(queryInput); + if (!candidate || candidate === activeQuery) return; + const timeout = window.setTimeout(() => { + setQueryAttempt(0); + setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); + setActiveQuery(candidate); + }, GRAPH_QUERY_DEBOUNCE_MILLISECONDS); + return () => window.clearTimeout(timeout); + }, [activeQuery, queryInput]); + + useEffect(() => { + if (!repository || !activeQuery) { + setQueryGraph(undefined); + setQueryLoading(false); + setQueryError(''); + return; + } + const expectedSnapshotId = repository.snapshot.id; + const expectedQuery = activeQuery; + const request = queryRequestGate.current.request({expectedQuery, expectedSnapshotId, scope: queryScope}, signal => + props.loadQuery(repository.id, expectedSnapshotId, expectedQuery, queryWorkingSet, signal), + ); + setQueryGraph(undefined); + setQueryLoading(true); + setQueryError(''); + setSelectedNodeId(undefined); + setExpandedNeighborhood(undefined); + setFocusRequest(undefined); + setFocusMode('all'); + setRelationFilter('all'); + setSizeMetric('connections'); + void request.result.then(outcome => { + if (!request.isCurrent()) return; + if (outcome.state === 'accepted') { + setQueryGraph(outcome.graph); + } else if (outcome.state === 'failed') { + setQueryGraph(undefined); + setQueryError(outcome.cause instanceof Error ? outcome.cause.message : String(outcome.cause)); + } + setQueryLoading(false); + }); + return () => { + request.cancel(); + }; + }, [activeQuery, props.loadQuery, queryScope, queryWorkingSet, repository?.id, repository?.snapshot.id]); + + useEffect(() => { + analysisAbortController.current?.abort(); + analysisRequestSequence.current += 1; + setAnalysis(undefined); + setAnalysisError(''); + setAnalysisLoading(false); + return () => { + analysisAbortController.current?.abort(); + analysisRequestSequence.current += 1; + }; + }, [repository?.id, repository?.snapshot.id]); + + const loadAnalysis = (): void => { + if (!repository || analysisLoading) return; + const requestedScope = analysisScope; + const requestSequence = analysisRequestSequence.current + 1; + analysisRequestSequence.current = requestSequence; + analysisAbortController.current?.abort(); + const controller = new AbortController(); + analysisAbortController.current = controller; + setAnalysisLoading(true); + setAnalysisError(''); + void props + .loadAnalysis(repository.id, repository.snapshot.id, controller.signal) + .then(next => { + if ( + graphAnalysisRequestIsCurrent( + analysisRequestSequence.current, + requestSequence, + analysisScopeRef.current, + requestedScope, + ) + ) { + setAnalysis(next); + } + }) + .catch(cause => { + if ( + !isAbortError(cause) && + graphAnalysisRequestIsCurrent( + analysisRequestSequence.current, + requestSequence, + analysisScopeRef.current, + requestedScope, + ) + ) { + setAnalysisError(cause instanceof Error ? cause.message : String(cause)); + } + }) + .finally(() => { + if ( + graphAnalysisRequestIsCurrent( + analysisRequestSequence.current, + requestSequence, + analysisScopeRef.current, + requestedScope, + ) + ) { + setAnalysisLoading(false); + } + }); + }; + + useEffect(() => { + if (!selectedNode || selectedNode.type !== 'symbol' || !repository) { + setNodeDetail(undefined); + setNodeDetailLoading(false); + setNodeDetailError(''); + return; + } + const key = `${repository.id}:${graph?.repository.snapshot.id ?? ''}:${selectedNode.id}`; + const cached = nodeDetailCache.current.get(key); + if (cached) { + cacheGraphNodeDetail(nodeDetailCache.current, key, cached); + setNodeDetail(cached); + setExpandedNeighborhood(cached); + setNodeDetailLoading(false); + setNodeDetailError(''); + focusSequence.current += 1; + setFocusRequest({nodeId: cached.node.id, sequence: focusSequence.current}); + return; + } + const controller = new AbortController(); + setNodeDetail(undefined); + setNodeDetailLoading(true); + setNodeDetailError(''); + void props + .loadNodeDetail(repository.id, repository.snapshot.id, selectedNode.id, controller.signal) + .then(detail => { + if ( + !graphNodeDetailRequestIsCurrent(controller.signal.aborted, detail, repository.snapshot.id, selectedNode.id) + ) + return; + cacheGraphNodeDetail(nodeDetailCache.current, key, detail); + setNodeDetail(detail); + setExpandedNeighborhood(detail); + focusSequence.current += 1; + setFocusRequest({nodeId: detail.node.id, sequence: focusSequence.current}); + }) + .catch(cause => { + if (!controller.signal.aborted && !isAbortError(cause)) { + setExpandedNeighborhood(undefined); + setNodeDetailError(cause instanceof Error ? cause.message : String(cause)); + } + }) + .finally(() => { + if (!controller.signal.aborted) setNodeDetailLoading(false); + }); + return () => { + controller.abort(); + }; + }, [graph?.repository.snapshot.id, props.loadNodeDetail, repository?.id, selectedNode?.id, selectedNode?.type]); + + const searchResults = useMemo(() => { + const needle = search.trim().toLowerCase(); + if (!needle || !graph) return []; + return graph.nodes + .filter( + node => + node.label.toLowerCase().includes(needle) || + node.qualifiedName?.toLowerCase().includes(needle) || + node.path?.toLowerCase().includes(needle), + ) + .sort((left, right) => right.degree - left.degree || compareCodeUnits(left.label, right.label)) + .slice(0, 8); + }, [graph, search]); + + const chooseRepository = (nextRepositoryId: string): void => { + const next = repositories.find(candidate => candidate.id === nextRepositoryId); + setRepositoryId(nextRepositoryId); + setViewId(next?.defaultViewId ?? next?.views[0]?.id ?? ''); + setProjectId('all'); + setWorkingSet(DEFAULT_WORKING_SET); + clearCatalogSearch(); + clearCodeQuery(); + }; + + const chooseView = (nextViewId: string): void => { + setViewId(nextViewId); + setProjectId('all'); + setWorkingSet(DEFAULT_WORKING_SET); + clearCatalogSearch(); + clearCodeQuery(); + }; + + const chooseCatalogView = (nextRepositoryId: string, nextViewId: string): void => { + setRepositoryId(nextRepositoryId); + setViewId(nextViewId); + setProjectId('all'); + setWorkingSet(DEFAULT_WORKING_SET); + clearCatalogSearch(); + clearCodeQuery(); + }; + + const chooseProject = (nextProjectId: string): void => { + setProjectId(nextProjectId); + setWorkingSet(DEFAULT_WORKING_SET); + setSearch(''); + setSelectedNodeId(undefined); + setExpandedNeighborhood(undefined); + clearCatalogSearch(); + clearCodeQuery(); + }; + + function clearCatalogSearch(): void { + setCatalogQuery(''); + setCatalogSearchResult(undefined); + setCatalogError(''); + } + + const submitCodeQuery = (): void => { + const candidate = managerGraphQueryCandidate(queryInput); + if (!candidate) { + setQueryError(`Enter between 1 and ${GRAPH_QUERY_MAXIMUM_LENGTH} characters to search the code graph.`); + return; + } + if (candidate === activeQuery) { + setQueryAttempt(current => current + 1); + return; + } + setQueryAttempt(0); + setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); + setActiveQuery(candidate); + }; + + function clearCodeQuery(): void { + queryRequestGate.current.cancelCurrent(); + setQueryInput(''); + setActiveQuery(''); + setQueryGraph(undefined); + setQueryLoading(false); + setQueryError(''); + setQueryAttempt(0); + setQueryWorkingSet(DEFAULT_QUERY_WORKING_SET); + setSelectedNodeId(undefined); + setExpandedNeighborhood(undefined); + setFocusRequest(undefined); + } + + const selectNode = (nodeId: string | undefined, focus = false): void => { + setSelectedNodeId(nodeId); + if (!nodeId) { + setFocusMode('all'); + setExpandedNeighborhood(undefined); + return; + } + if (baseGraph?.nodes.some(node => node.id === nodeId)) setExpandedNeighborhood(undefined); + if (focus) { + focusSequence.current += 1; + setFocusRequest({nodeId, sequence: focusSequence.current}); + } + }; + + const loadCatalogContinuation = (requestedQuery: string): void => { + if (!repository || !repositoryGroup || catalogLoading) return; + const query = requestedQuery.trim().slice(0, 256); + const continuation = catalogContinuation?.viewId === repository.id ? catalogContinuation : undefined; + const offsets = + query.length === 0 + ? graphCatalogPageOffsets({ + baseRepository, + baseRepositoryGroup, + checkoutId: repository.checkoutId, + continuation, + viewId: repository.id, + }) + : {projectOffset: 0, viewOffset: 0, workspaceOffset: 0}; + const {projectOffset, viewOffset, workspaceOffset} = offsets; + const requestedScope = `${repository.id}:${repository.snapshot.id}:${projectOffset}:${workspaceOffset}:${viewOffset}:${query}`; + const requestSequence = catalogRequestSequence.current + 1; + catalogRequestSequence.current = requestSequence; + catalogAbortController.current?.abort(); + const controller = new AbortController(); + catalogAbortController.current = controller; + setCatalogLoading(true); + setCatalogError(''); + void Promise.all([ + props.loadCatalogPage( + repository.id, + repository.snapshot.id, + projectOffset, + workspaceOffset, + query, + controller.signal, + ), + props.loadViewsPage(repository.id, viewOffset, query, controller.signal), + ]) + .then(([catalogPage, viewPage]) => { + const currentScope = `${repository.id}:${repository.snapshot.id}:${projectOffset}:${workspaceOffset}:${viewOffset}:${query}`; + if ( + controller.signal.aborted || + catalogRequestSequence.current !== requestSequence || + currentScope !== requestedScope + ) + return; + const selectedViewGroup: GraphRepositoryGroup = { + ...repositoryGroup, + defaultViewId: repositoryGroup.defaultViewId, + views: [catalogPage.repository], + viewsTruncated: false, + }; + setCatalogAdditions(current => + mergeGraphRepositoryGroups(current, [selectedViewGroup, ...viewPage.repositories]), + ); + if (query.length > 0) { + setCatalogSearchResult({ + options: graphCatalogSearchOptions(catalogPage.repository, viewPage.repositories), + query, + }); + } + if (query.length === 0) { + setCatalogContinuation({ + projectHasMore: catalogPage.repository.projectsTruncated, + projectOffset: + projectOffset + catalogPage.repository.projects.filter(project => project.id.startsWith('cgp_')).length, + viewHasMore: viewPage.hasMore, + viewId: repository.id, + viewOffset: + viewOffset + + viewPage.repositories + .flatMap(group => group.views) + .filter(view => view.checkoutId === repository.checkoutId).length, + workspaceHasMore: catalogPage.repository.workspacesTruncated, + workspaceOffset: workspaceOffset + catalogPage.repository.workspaces.length, + }); + } + }) + .catch(cause => { + if (!controller.signal.aborted && catalogRequestSequence.current === requestSequence) { + setCatalogError(cause instanceof Error ? cause.message : String(cause)); + } + }) + .finally(() => { + if (!controller.signal.aborted && catalogRequestSequence.current === requestSequence) setCatalogLoading(false); + }); + }; + + useEffect(() => { + setCatalogAdditions([]); + }, [baseCatalogIdentity]); + + useEffect(() => { + catalogAbortController.current?.abort(); + catalogRequestSequence.current += 1; + setCatalogContinuation(undefined); + setCatalogError(''); + setCatalogSearchResult(undefined); + setCatalogLoading(false); + }, [baseCatalogIdentity, repository?.id, repository?.snapshot.id]); + + return ( +
+
+
+

Native code intelligence

+

Knowledge graph

+

+ Explore architecture from repository-level structure down to individual symbols. +

+
+ +
+ +
+ undefined)} + onDiagnostics={props.onDiagnostics ?? (() => undefined)} + output={props.administrationOutput} + report={props.administration} + /> + {props.catalog?.automaticCompaction ? ( + + ) : null} + {props.catalog?.maintenance ? ( + + ) : null} + {activeBuilds.length > 0 ? ( +
+ {activeBuilds.map(build => ( + + ))} +
+ ) : null} + + {props.catalog?.diagnostics.length ? ( +
+ Some indexed views need attention + {props.catalog.diagnostics.map(diagnostic => ( + {diagnostic.message} + ))} +
+ ) : null} +
+ +
+
+ + {repositoryGroup && (repositoryGroup.views.length > 1 || viewCatalogHasMore) ? ( + + ) : null} + +
+
+ +
+ { + setCatalogQuery(event.target.value); + setCatalogSearchResult(undefined); + setCatalogError(''); + }} + onKeyDown={event => { + if (event.key !== 'Enter') return; + event.preventDefault(); + loadCatalogContinuation(catalogQuery); + }} + placeholder="Component, workspace, commit, or view" + type="search" + value={catalogQuery} + /> + +
+ {projectCatalogHasMore || workspaceCatalogHasMore || viewCatalogHasMore ? ( + + ) : null} + {catalogError ? {catalogError} : null} + {catalogSearchResult ? ( +
+ {catalogSearchResult.options.projects.length + catalogSearchResult.options.views.length > 0 ? ( + <> +

+ Found{' '} + {( + catalogSearchResult.options.projects.length + catalogSearchResult.options.views.length + ).toLocaleString()}{' '} + options for “{catalogSearchResult.query}” +

+ {catalogSearchResult.options.projects.length > 0 ? ( +
+ Components and workspace matches + {catalogSearchResult.options.projects.map(option => ( + + ))} +
+ ) : null} + {catalogSearchResult.options.views.length > 0 ? ( +
+ Indexed views + {catalogSearchResult.options.views.map(option => ( + + ))} +
+ ) : null} + + ) : ( +

No catalog matches for “{catalogSearchResult.query}”

+ )} +
+ ) : ( + + Search results appear here. + + )} +
+
+ + setSearch(event.target.value)} + placeholder={graph?.mode === 'overview' ? 'Search components' : 'Name, path, or symbol'} + type="search" + value={search} + /> + {search.trim() ? ( +
+ {searchResults.length > 0 ? ( + searchResults.map(node => ( + + )) + ) : ( +

No matching nodes

+ )} +
+ ) : null} +
+
+ +
+ setQueryInput(event.target.value)} + onKeyDown={event => { + if (event.key !== 'Enter') return; + event.preventDefault(); + submitCodeQuery(); + }} + placeholder="Concept, path, module, or symbol" + type="search" + value={queryInput} + /> + +
+ {activeQuery ? ( + + ) : null} + {!activeQuery && queryError ? {queryError} : null} +
+
+ {graph ? compactNumber(graph.stats.renderedNodes) : '—'} nodes + {graph ? compactNumber(graph.stats.renderedEdges) : '—'} links + {graph?.paging.hasMore ? ( + + ) : null} + WebGL +
+
+ + {graph ? ( +
+ + {graph.mode === 'detail' ? ( + + ) : ( +
+ Node size + {graphOverviewSizeLabel(graph)} +
+ )} +
+ Selection focus +
+ {( + [ + ['all', 'All'], + ['neighbors', 'Neighbors'], + ['incoming', 'Incoming'], + ['outgoing', 'Outgoing'], + ] as const + ).map(([mode, label]) => ( + + ))} +
+
+ {selectedNode ? ( + + ) : ( +

Select a node to isolate its neighborhood and direction.

+ )} +
+ ) : null} + +
+
+ {!props.catalog && props.catalogError ? ( +
+
+ ) : !props.catalog ? ( +
+
+ ) : repositories.length === 0 ? ( + build.state === 'queued' || build.state === 'running')} + /> + ) : activeQuery && selectedRepositoryIsIndexing && !queryGraph ? ( +
+
+ ) : activeQuery && queryError ? ( +
+
+ ) : !activeQuery && error ? ( +
+
+ ) : (activeQuery ? queryLoading : loading) || !graph ? ( +
+
+ ) : activeQuery && (graph.query?.matchedNodes === 0 || graph.nodes.length === 0) ? ( +
+
+ ) : ( + selectNode(nodeId, Boolean(nodeId))} + relationFilter={relationFilter} + sizeMetric={sizeMetric} + focusRequest={focusRequest} + selectedNodeId={selectedNodeId} + /> + )} +
+ + +
+ + {graph && [...new Set([...graph.warnings, ...(graph.query?.warnings ?? [])])].length ? ( +
+ {[...new Set([...graph.warnings, ...(graph.query?.warnings ?? [])])].map(warning => ( + {warning} + ))} +
+ ) : null} +
+ ); +} diff --git a/src/manager_request_inputs.ts b/src/manager_request_inputs.ts new file mode 100644 index 00000000..4061dc1f --- /dev/null +++ b/src/manager_request_inputs.ts @@ -0,0 +1,80 @@ +import {Option} from 'effect'; +import type {ConsolidationAgent, MemoryKind, MemoryStatus} from './types.js'; + +class ManagerRequestInputError extends Error { + override readonly name = 'ManagerRequestInputError'; +} + +export function requiredQuery(url: URL, name: string): string { + const value = url.searchParams.get(name); + if (!value) throw new ManagerRequestInputError(`Missing query parameter: ${name}`); + return value; +} + +export function optionalPositiveIntegerQuery(url: URL, name: string): Option.Option { + return Option.fromNullishOr(url.searchParams.get(name)).pipe( + Option.map(value => Number(value)), + Option.filter(value => Number.isSafeInteger(value) && value > 0), + ); +} + +export function optionalNonNegativeIntegerQuery(url: URL, name: string): Option.Option { + return Option.fromNullishOr(url.searchParams.get(name)).pipe( + Option.map(value => Number(value)), + Option.filter(value => Number.isSafeInteger(value) && value >= 0), + ); +} + +export function optionalNonEmptyQuery(url: URL, name: string): Option.Option { + return Option.fromNullishOr(url.searchParams.get(name)).pipe( + Option.map(value => value.trim()), + Option.filter(value => value.length > 0), + ); +} + +export function requireString(value: unknown, name: string): string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ManagerRequestInputError(`Provide ${name}.`); + } + return value; +} + +export function optionalString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value : undefined; +} + +export function requireStringArray(value: unknown, name: string): readonly string[] { + if (!Array.isArray(value) || value.length === 0 || !value.every(item => typeof item === 'string')) { + throw new ManagerRequestInputError(`Provide ${name} as a non-empty string array.`); + } + return value; +} + +export function requireConfirm(body: Record): void { + if (body.confirm !== true) throw new ManagerRequestInputError('Set confirm=true for this action.'); +} + +export function memoryKind(value: unknown): MemoryKind | undefined { + return value === 'durable' || + value === 'handoff' || + value === 'incident' || + value === 'preference' || + value === 'smoke' + ? value + : undefined; +} + +export function memoryStatus(value: unknown): MemoryStatus | undefined { + return value === 'active' || value === 'archived' || value === 'superseded' ? value : undefined; +} + +export function consolidationAgent(value: string): ConsolidationAgent { + if (value === 'codex' || value === 'claude' || value === 'cursor' || value === 'copilot' || value === 'effect-ai') { + return value; + } + throw new ManagerRequestInputError(`Unsupported consolidation agent: ${value}`); +} + +export function cleanupMode(value: unknown): 'archive' | 'forget' | 'keep' { + return value === 'forget' || value === 'keep' ? value : 'archive'; +} diff --git a/src/manager_ui.tsx b/src/manager_ui.tsx index bd72f26c..a5d552d4 100644 --- a/src/manager_ui.tsx +++ b/src/manager_ui.tsx @@ -4,6 +4,7 @@ import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import type {CodeGraphLocalDiagnosticsReport} from './code_graph/diagnostics.js'; import {ManagerAutocompleteInput, ManagerDialogProvider, useManagerDialogs} from './manager_dialog.js'; +import {WorksetsPanel} from './manager_worksets_view.js'; import { graphViewRemovalApprovalDialog, graphViewRemovalTargetIsAbsent, @@ -19,18 +20,52 @@ import { graphMaintenanceStatusLabel, graphStatusPollDelay, graphStatusRequiresCatalogRefresh, - type GraphAnalysis, + mergeGraphCatalogStatus, type GraphAdministrationAction, type GraphCatalog, - type GraphCatalogPage, - type GraphNodeDetail, - type GraphQueryVisualization, - type GraphVisualization, - type GraphViewPage, } from './manager_graph.js'; -import type {ManagerGraphVisualizationLimits} from './manager_graph_limits.js'; - -type PanelName = 'doctor' | 'graph' | 'memory' | 'shares' | 'tools'; +import { + GRAPH_CATALOG_REQUEST_TIMEOUT_MILLISECONDS, + SharesPanel, + actionProgressLabel, + api, + bulkActionLabel, + countFiles, + errorMessage, + findNodeInTrees, + formatBulkResults, + graphAdministrationActionLabel, + isMarkdownNode, + isMarkdownUri, + isResourceUri, + loadManagerGraph, + loadManagerGraphAnalysis, + loadManagerGraphCatalogPage, + loadManagerGraphNodeDetail, + loadManagerGraphQuery, + loadManagerGraphViewsPage, + managerProjectOptions, + markdownBodyForPreview, + nodeMatches, + panelDescription, + panelIcon, + panelNavDescription, + pruneSelectedMemoryUris, + resourceUrisFromText, + selectableMemoryUris, + tabTitle, + treeItemClass, + uniqueSelectorValues, +} from './manager_ui_support.js'; + +export { + graphAdministrationActionLabel, + managerProjectOptions, + pruneSelectedMemoryUris, + selectableMemoryUris, +} from './manager_ui_support.js'; + +export type PanelName = 'doctor' | 'graph' | 'memory' | 'shares' | 'tools' | 'worksets'; type NavTreeTab = 'memories' | 'resources'; type CheckStatus = 'fail' | 'ok' | 'warn'; type MemoryKind = 'durable' | 'handoff' | 'incident' | 'preference' | 'smoke'; @@ -109,7 +144,7 @@ interface StateResponse { readonly version: string; } -interface ShareSummary { +export interface ShareSummary { readonly addedAt: string; readonly ahead?: number; readonly behind?: number; @@ -138,7 +173,7 @@ interface ConsolidationJob { readonly status: 'completed' | 'failed' | 'running'; } -interface BulkItemResult { +export interface BulkItemResult { readonly error?: string; readonly ok: boolean; readonly output?: string; @@ -159,10 +194,7 @@ interface DropdownOption { readonly value: string; } -const token = typeof window === 'undefined' ? '' : (new URLSearchParams(window.location.search).get('token') ?? ''); const EMPTY_SELECTED_URIS: ReadonlySet = new Set(); -const GRAPH_CATALOG_REQUEST_TIMEOUT_MILLISECONDS = 10_000; -const GRAPH_DETAIL_REQUEST_TIMEOUT_MILLISECONDS = 30_000; function clampSidebarWidth(width: number): number { return Math.min(SIDEBAR_WIDTH_MAX, Math.max(SIDEBAR_WIDTH_MIN, Math.round(width))); @@ -255,7 +287,13 @@ function App(): React.ReactElement { const status = await api< Pick< GraphCatalog, - 'builds' | 'catalogRevision' | 'lifecyclePending' | 'maintenance' | 'waiterCount' | 'waiters' + | 'automaticCompaction' + | 'builds' + | 'catalogRevision' + | 'lifecyclePending' + | 'maintenance' + | 'waiterCount' + | 'waiters' > >('/api/graphs/status', undefined, {timeoutMilliseconds: GRAPH_CATALOG_REQUEST_TIMEOUT_MILLISECONDS}); if (cancelled) return; @@ -276,15 +314,16 @@ function App(): React.ReactElement { timeoutMilliseconds: GRAPH_CATALOG_REQUEST_TIMEOUT_MILLISECONDS, }); if (cancelled) return; - graphCatalogRef.current = refreshed; - setGraphCatalog(refreshed); + const refreshedWithStatus = mergeGraphCatalogStatus(refreshed, status); + graphCatalogRef.current = refreshedWithStatus; + setGraphCatalog(refreshedWithStatus); setGraphCatalogError(''); for (const build of status.builds) { const identity = graphCompletedBuildResultIdentity(build); if (identity) acknowledgedCompletedResults.add(identity); } - } else if (graphCatalogRef.current) { - const merged = {...graphCatalogRef.current, ...status}; + } else { + const merged = mergeGraphCatalogStatus(graphCatalogRef.current, status); graphCatalogRef.current = merged; setGraphCatalog(merged); } @@ -302,7 +341,7 @@ function App(): React.ReactElement { observedActiveMaintenance = activeMaintenance; timer = window.setTimeout( () => void poll(), - graphStatusPollDelay(status.builds, status.maintenance, status.lifecyclePending), + graphStatusPollDelay(status.builds, status.maintenance, status.lifecyclePending, status.automaticCompaction), ); } catch { if (!cancelled) timer = window.setTimeout(() => void poll(), 15_000); @@ -1108,7 +1147,7 @@ function App(): React.ReactElement {

Workspace