diff --git a/CHANGELOG.md b/CHANGELOG.md index c9b68b091..29cd53860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- `codegraph_explore` finds code containing storage keys, command-line flags, and event names; rebuild existing indexes to enable these matches. + - **Codex and Astra read project guidance from `AGENTS.md`.** The canonical agent guide now lives in `AGENTS.md` (with a nested `docs/AGENTS.md` for long validation notes); `CLAUDE.md` is a thin `@AGENTS.md` wrapper for Claude Code. Codex/Astra no longer miss the old CLAUDE-only instructions. - **A big screen's picture stops wrapping into a column.** How wide a screen's lines run before they wrap was worked out with a formula, and the formula was wrong for the way these pictures are actually drawn: a part of a screen spends lines on its own structure — a step that fires things gets a line to itself, and what it fires starts another — so estimating the lines from the boxes alone badly undercounted them, and one screen's 98 boxes wrapped into a 4,356px column. Laying a picture out is cheap and exact, so the widths are now simply tried and the one that comes out closest to the shape of a window is kept. Across one app's 51 screens the tallest picture went from 4,356px to 3,796px, total height fell 8%, and — because a shorter picture is also a picture whose lines have less far to go — lines running over other boxes fell by a third and lines crossing each other went from 13 to 5. diff --git a/__tests__/kernel-retry-materialize.test.ts b/__tests__/kernel-retry-materialize.test.ts index 23965bd2d..83e4db423 100644 --- a/__tests__/kernel-retry-materialize.test.ts +++ b/__tests__/kernel-retry-materialize.test.ts @@ -56,7 +56,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { it('storeExtractionResult persists the decoded nodes of a raw kernel result', async () => { const source = 'def target_fn(root, mission_path):\n' + - ' return (root, mission_path)\n' + + ' return (root, mission_path, "adapter.mission")\n' + '\n' + 'class Adapter:\n' + ' def adapt(self):\n' + @@ -93,6 +93,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { expect(nodes.length).toBe(raw!.counts.nodes); expect(nodes.map((n) => n.name)).toContain('target_fn'); expect(nodes.map((n) => n.name)).toContain('Adapter'); + expect(cg.findLiteralSeedIds('"adapter.mission"').map(id => cg.getNode(id)?.name)).toEqual(['target_fn']); }); }); diff --git a/__tests__/literal-capture.test.ts b/__tests__/literal-capture.test.ts new file mode 100644 index 000000000..a27406a51 --- /dev/null +++ b/__tests__/literal-capture.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import type { Node } from '../src/types'; +import { captureLiterals, isSeedLiteral, seedLiteralsInQuery } from '../src/extraction/literal-capture'; + +function node(id: string, kind: Node['kind'], startLine: number, endLine: number): Node { + return { + id, kind, name: id, qualifiedName: id, filePath: 'src/a.ts', language: 'typescript', + startLine, endLine, startColumn: 0, endColumn: Number.MAX_SAFE_INTEGER, updatedAt: 0, + }; +} + +describe('isSeedLiteral', () => { + it('keeps storage keys, flags, dotted names and paths', () => { + for (const v of ['bompus_custom_ds_players', '--start', '-v', 'draft.pick', 'api/v1/users', 'ns:event']) + expect(isSeedLiteral(v), v).toBe(v !== '-v'); + }); + it('drops plain words, prose, and values that start with a separator', () => { + for (const v of ['ready', 'Error', 'not found', './utils', '../x', '', 'a_b']) + expect(isSeedLiteral(v), v).toBe(false); + }); +}); + +describe('seedLiteralsInQuery', () => { + it('finds quoted spans and bare runs, stripping surrounding punctuation', () => { + expect(seedLiteralsInQuery('who writes "bompus_custom_ds_players" (via --start)?')) + .toEqual(['bompus_custom_ds_players', '--start']); + }); + it('returns nothing for a symbol-anchored question', () => { + expect(seedLiteralsInQuery('callers of espnPlayerKey in shared')).toEqual([]); + }); +}); + +describe('captureLiterals', () => { + const source = [ + `import { x } from './utils/helpers';`, // 1: path starts with '.', never qualifies + `const KEY = 'bompus_custom_ds_players';`, // 2: top level → file node + `export function save() {`, // 3 + ` storage.set('bompus_custom_ds_players', 1);`, // 4 + ` log('ready');`, // 5: plain word + ` emit(\`draft.pick\`); emit(\`draft.\${n}\`);`, // 6: second one interpolates + `}`, // 7 + `export class Boot { start() { run('--start'); } }`,// 8 + ].join('\n'); + + it('attributes each literal to the innermost enclosing symbol, else the file', () => { + const file = node('file:src/a.ts', 'file', 1, 8); + const save = node('save', 'function', 3, 7); + const boot = node('Boot', 'class', 8, 8); + const start = node('start', 'method', 8, 8); + const imp = node('./utils/helpers', 'import', 1, 1); + const nodes = [file, imp, save, boot, start]; + captureLiterals(source, nodes); + expect(file.literals).toEqual(['bompus_custom_ds_players']); + expect(save.literals).toEqual(['bompus_custom_ds_players', 'draft.pick']); + expect(start.literals).toEqual(['--start']); + expect(boot.literals).toBeUndefined(); + expect(imp.literals).toBeUndefined(); + }); + + it('dedupes per node and caps at 32', () => { + const many = Array.from({ length: 40 }, (_, i) => `k('key_${i}'); k('key_${i}');`).join('\n'); + const fn = node('f', 'function', 1, 40); + captureLiterals(many, [node('file:src/a.ts', 'file', 1, 40), fn]); + expect(fn.literals).toHaveLength(32); + expect(new Set(fn.literals).size).toBe(32); + }); + + it('uses UTF-16 columns to distinguish same-line siblings after non-ASCII source', () => { + const prefix = '/* café 😀 */ '; + const first = "function writer(){return 'cache.write';}"; + const second = "function reader(){return 'cache.read';}"; + const writer = { ...node('writer', 'function', 1, 1), startColumn: prefix.length, endColumn: (prefix + first).length }; + const reader = { ...node('reader', 'function', 1, 1), startColumn: (prefix + first).length, endColumn: (prefix + first + second).length }; + captureLiterals(prefix + first + second, [writer, reader]); + expect(writer.literals).toEqual(['cache.write']); + expect(reader.literals).toEqual(['cache.read']); + }); +}); diff --git a/__tests__/literal-compiled-index.test.ts b/__tests__/literal-compiled-index.test.ts new file mode 100644 index 000000000..ceb39d958 --- /dev/null +++ b/__tests__/literal-compiled-index.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import type CodeGraph from '../src'; +import type { QueryBuilder } from '../src/db/queries'; + +const built = path.join(__dirname, '..', 'dist', 'index.js'); +const kernel = path.join(__dirname, '..', 'codegraph-kernel', 'prebuilds', `${process.platform}-${process.arch}`, 'codegraph-kernel.node'); + +// Source-mode suites cannot exercise the compiled parse/store worker boundary. +describe.runIf(fs.existsSync(built))('literal persistence through compiled indexing', () => { + let dir: string | undefined; + let cg: CodeGraph | undefined; + afterEach(() => { + cg?.destroy(); + cg = undefined; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + dir = undefined; + vi.unstubAllEnvs(); + }); + + for (const mode of ['native-worker', 'native-main', 'wasm-worker']) { + it.runIf(mode === 'wasm-worker' || fs.existsSync(kernel))(`${mode}: fresh, unchanged, and migrated indexes retain exact owners`, async () => { + vi.stubEnv('CODEGRAPH_PARSE_WORKERS', '1'); + vi.stubEnv('CODEGRAPH_KERNEL', mode === 'wasm-worker' ? '0' : '1'); + vi.stubEnv('CODEGRAPH_NO_STORE_WORKER', mode === 'native-main' ? '1' : '0'); + const BuiltCodeGraph: typeof CodeGraph = require(built).default; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literal-compiled-')); + fs.writeFileSync(path.join(dir, 'cache.py'), "def persist():\n return 'cache.write'\n"); + fs.writeFileSync(path.join(dir, 'siblings.ts'), "/* café 😀 */ export function writer(){return 'sibling.write';} export function reader(){return 'sibling.read';}\n"); + cg = await BuiltCodeGraph.init(dir, { silent: true }); + const names = (key: string) => cg!.findLiteralSeedIds(key).map(id => cg!.getNode(id)?.name); + const assertOwners = () => { + expect(names('cache.write')).toEqual(['persist']); + expect(names('sibling.write')).toEqual(['writer']); + expect(names('sibling.read')).toEqual(['reader']); + }; + await cg.indexAll(); + assertOwners(); + await cg.indexAll(); + assertOwners(); + + // A schema-only migration has no literal data; unchanged source still needs backfill. + const queries = (cg as unknown as { queries: QueryBuilder }).queries; + queries.replaceLiteralsForFile('cache.py', []); + queries.replaceLiteralsForFile('siblings.ts', []); + queries.setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + await cg.indexAll(); + assertOwners(); + expect(cg.isIndexStale()).toBe(false); + fs.rmSync(path.join(dir, 'cache.py')); + await cg.indexAll(); + expect(names('cache.write')).toEqual([]); + expect(names('sibling.write')).toEqual(['writer']); + }); + } +}); diff --git a/__tests__/literal-seeds.test.ts b/__tests__/literal-seeds.test.ts new file mode 100644 index 000000000..992a1acb9 --- /dev/null +++ b/__tests__/literal-seeds.test.ts @@ -0,0 +1,253 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; +import { createDatabase, type SqliteDatabase } from '../src/db/sqlite-adapter'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; +import { ToolHandler } from '../src/mcp/tools'; + +/** + * The string-anchored explore path: a query that names a storage key or a CLI + * flag reaches the symbols whose bodies hold it, through the `literals` side + * table (written on the node write path, deleted with the file, cleared by a + * full index), and those symbols lead the ranking. + */ +describe('literal seeds — index, query, and lifecycle', () => { + let dir: string; + let cg: CodeGraph; + const queries = () => (cg as unknown as { queries: QueryBuilder }).queries; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literals-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'writer.ts'), + `export function persistRows(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +export function unrelatedHelper() { return 'ready'; } +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'reader.ts'), + `export async function readRows() { + const got = await chrome.storage.local.get('bompus_custom_ds_players'); + return got; +} +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'cli.ts'), + `export function main(argv: string[]) { + if (argv.includes('--start')) return start(); + return 0; +} +function start() { return 1; } +`, + ); + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('resolves a storage key to every symbol whose body holds it', () => { + const ids = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']); + const names = ids.map((id) => cg.getNode(id)?.name).sort(); + expect(names).toEqual(['persistRows', 'readRows']); + }); + + it('recommends rebuilding an index from before literal capture', () => { + const current = queries().getMetadata('indexed_with_extraction_version'); + try { + queries().setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + } finally { + queries().setMetadata('indexed_with_extraction_version', current!); + } + expect(cg.isIndexStale()).toBe(false); + }); + + it('a quoted key in a prose query puts the writers at the top of the subgraph', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootNames = sub.roots.map((id) => sub.nodes.get(id)?.name); + expect(rootNames.slice(0, 2).sort()).toEqual(['persistRows', 'readRows']); + expect(rootNames).not.toContain('unrelatedHelper'); + }); + + it('a bare CLI flag seeds too', () => { + const ids = queries().findNodeIdsByLiteral(['--start']); + expect(ids.map((id) => cg.getNode(id)?.name)).toEqual(['main']); + }); + + it('CODEGRAPH_LITERAL_SEEDS=0 turns the seed off (the ablation switch)', async () => { + process.env.CODEGRAPH_LITERAL_SEEDS = '0'; + try { + const sub = await cg.findRelevantContext('bompus_custom_ds_players'); + const names = [...sub.nodes.values()].map((n) => n.name); + expect(names).not.toContain('persistRows'); + } finally { + delete process.env.CODEGRAPH_LITERAL_SEEDS; + } + }); + + it("a file's rows leave with its nodes on re-index, and a full index clears the table", async () => { + fs.writeFileSync(path.join(dir, 'src', 'reader.ts'), `export function readRows() { return null; }\n`); + await cg.indexFiles(['src/reader.ts']); + let names = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']).map((id) => cg.getNode(id)?.name); + expect(names).toEqual(['persistRows']); + + fs.rmSync(path.join(dir, 'src', 'writer.ts')); + await cg.indexAll(); + expect(queries().findNodeIdsByLiteral(['bompus_custom_ds_players'])).toEqual([]); + }); +}); + +/** + * A literal held through a constant lives in a small file with no callers. In explore's file + * sort that file must take a source slot the way a file defining a named symbol does; on graph + * centrality alone it loses the slot to a hub file that never mentions the literal. + */ +describe('literal seeds — explore renders the holder file', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litrender-')); + fs.mkdirSync(path.join(dir, 'shared'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'callers'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'shared', 'bag.ts'), + `const BAG_KEY = 'bompus_custom_ds_players'; +export function persistBag(rows: unknown[]) { + chrome.storage.local.set({ [BAG_KEY]: rows }); +} +`, + ); + // The hub: matches the query word "storage" by name and is called from many files. + fs.writeFileSync( + path.join(dir, 'shared', 'storage.ts'), + `export function storageSet(key: string, value: unknown) { + return chrome.storage.local.set({ [key]: value }); +} +export function storageGet(key: string) { + return chrome.storage.local.get(key); +} +`, + ); + for (let i = 0; i < 8; i++) { + fs.writeFileSync( + path.join(dir, 'callers', `writer${i}.ts`), + `import { storageSet, storageGet } from '../shared/storage'; +export function storageWriter${i}() { + storageGet('k${i}'); + return storageSet('k${i}', ${i}); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('the file holding the quoted key takes the first source slot', async () => { + const res = await handler.execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + maxFiles: 1, + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced).toEqual(['shared/bag.ts']); + }); +}); + +/** + * A key held in more files than the entry-point cap (`searchLimit`, 3 by default and 8 in + * explore) must reach every holder: the cap is for text matches, and the literal lookup is + * already bounded. Before this, holders competed for the cap by path order and `shared/` + * lost to `scripts/`. + */ +describe('literal seeds — every holder reaches the subgraph past the entry cap', () => { + let dir: string; + let cg: CodeGraph; + const N = 10; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litcap-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + for (let i = 0; i < N; i++) { + fs.writeFileSync( + path.join(dir, 'src', `holder${i}.ts`), + `export function write${i}(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('all holders are roots with the default searchLimit of 3', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootFiles = new Set(sub.roots.map((id) => sub.nodes.get(id)?.filePath)); + expect(rootFiles.size).toBe(N); + }); + + it('explore renders every holder with the default file cap', async () => { + const res = await new ToolHandler(cg).execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced.length).toBe(N); + }); +}); + +describe('literals — v10 migration', () => { + let dir: string; + let db: SqliteDatabase | null = null; + + afterEach(() => { + db?.close(); + db = null; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function makeV9Db(): SqliteDatabase { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litmigrate-')); + const conn = createDatabase(path.join(dir, 'legacy.db')).db; + conn.exec(` + CREATE TABLE schema_versions (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL, description TEXT); + INSERT INTO schema_versions VALUES (9, 0, 'legacy'); + `); + db = conn; + return conn; + } + + it('creates the empty table and is idempotent on replay', () => { + const conn = makeV9Db(); + runMigrations(conn, 9); + expect(getCurrentVersion(conn)).toBe(CURRENT_SCHEMA_VERSION); + expect((conn.prepare('SELECT COUNT(*) AS n FROM literals').get() as { n: number }).n).toBe(0); + conn.prepare('DELETE FROM schema_versions WHERE version >= 10').run(); + expect(() => runMigrations(conn, 9)).not.toThrow(); + }); +}); diff --git a/src/context/index.ts b/src/context/index.ts index c819a55ca..95720edf9 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -172,8 +172,16 @@ const DEFAULT_FIND_OPTIONS: Required = { edgeKinds: [], nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default seedNames: [], // Segment-vocab supplement — filled by the facade + seedNodeIds: [], // Literal-table seeds — filled by the facade }; +/** + * Score for a literal seed. Above the BM25 magnitudes the FTS channel returns + * (tens to hundreds) and every additive boost below, so a symbol holding the + * quoted literal leads the ranking whatever else the query matched. + */ +const LITERAL_SEED_SCORE = 1000; + // Re-export the low-confidence sentinel (defined in a dependency-free leaf so // the MCP layer can import it without pulling this module's deps onto the // cold-start path). Builder code below uses the imported binding directly. @@ -483,7 +491,17 @@ export class ContextBuilder { // Step 2: Look up exact matches for extracted symbols let exactMatches: SearchResult[] = []; - if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0) { + // Literal seeds bypass every size trim below: the lookup already bounds + // them, and a storage key held in eleven files must reach all eleven, not + // the first `searchLimit` by path order. + const literalSeedIds = new Set(opts.seedNodeIds); + const keepSeedsThenTop = (results: SearchResult[], n: number): SearchResult[] => { + if (literalSeedIds.size === 0) return results.slice(0, n); + const seeds = results.filter((r) => literalSeedIds.has(r.node.id)); + const rest = results.filter((r) => !literalSeedIds.has(r.node.id)).slice(0, n); + return [...seeds, ...rest]; + }; + if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0 || opts.seedNodeIds.length > 0) { try { if (symbolsFromQuery.length > 0) { // Get more results so we can apply co-location boosting before trimming @@ -493,6 +511,20 @@ export class ContextBuilder { }); } + // Literal seeds: the query quoted a string that lives in these + // symbols' bodies. No name matches it, so they enter here, first and + // at a fixed top score; the co-location pass below still compounds + // several hits in one file. + if (opts.seedNodeIds.length > 0) { + const byId = this.queries.getNodesByIds(opts.seedNodeIds); + for (const id of opts.seedNodeIds) { + const node = byId.get(id); + if (!node || exactMatches.some((r) => r.node.id === id)) continue; + exactMatches.push({ node, score: LITERAL_SEED_SCORE }); + } + logDebug('Literal seed matches', { seedNodeIds: opts.seedNodeIds, added: byId.size }); + } + // Step 2a: segment-vocabulary seeds. Word-level query terms cannot // reach camelCase names through FTS (one token per name), so the // caller resolves query words → names via the segment vocab and hands @@ -538,7 +570,7 @@ export class ContextBuilder { } // Trim back to reasonable size - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 2)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 2)); logDebug('Exact symbol matches', { count: exactMatches.length }); } catch (error) { logDebug('Exact symbol lookup failed', { error: String(error) }); @@ -587,7 +619,7 @@ export class ContextBuilder { } } exactMatches.sort((a, b) => b.score - a.score); - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 3)); } // Step 3: Run text search for natural language term matching @@ -977,7 +1009,7 @@ export class ContextBuilder { // compound) have now contributed. Sort by score so multi-term matches from // later steps can outrank dampened single-term matches from earlier steps. searchResults.sort((a, b) => b.score - a.score); - searchResults = searchResults.slice(0, opts.searchLimit * 3); + searchResults = keepSeedsThenTop(searchResults, opts.searchLimit * 3); // Filter by minimum score let filteredResults = searchResults.filter((r) => r.score >= opts.minScore); @@ -991,7 +1023,7 @@ export class ContextBuilder { // With 36 entry points and maxNodes=120, each gets only 3 nodes — useless. // Cap to searchLimit so each entry point gets a meaningful traversal budget. if (filteredResults.length > opts.searchLimit) { - filteredResults = filteredResults.slice(0, opts.searchLimit); + filteredResults = keepSeedsThenTop(filteredResults, opts.searchLimit); } // Confidence signal for the honest-handoff footer (consumed in buildContext). diff --git a/src/db/migrations.ts b/src/db/migrations.ts index b6c82a5e5..b3f7f2083 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 9; +export const CURRENT_SCHEMA_VERSION = 10; /** * Migration definition @@ -177,6 +177,26 @@ const migrations: Migration[] = [ ); }, }, + { + version: 10, + description: + 'Add literals — identifier-like string literal → enclosing symbol, so explore seeds on storage keys and flags', + up: (db) => { + // DDL only. No backfill: the values come from file CONTENT the migration + // cannot see, so the table stays empty until the next full index and + // explore behaves exactly as before for those queries. Keep in lockstep + // with schema.sql. + db.exec(` + CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index 37c078845..dc489cacf 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -257,6 +257,7 @@ export class QueryBuilder { updateNode?: SqliteStatement; deleteNode?: SqliteStatement; deleteNodesByFile?: SqliteStatement; + deleteLiteralsByFile?: SqliteStatement; getNodeById?: SqliteStatement; getNodesByFile?: SqliteStatement; getNodesByKind?: SqliteStatement; @@ -476,6 +477,43 @@ export class QueryBuilder { // import-only name can never be surfaced (getSegmentMatches requires a // real definition), so its rows would only inflate the rarity statistics. if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); + const literalRows: unknown[][] = []; + this.collectLiteralRows(node, literalRows); + this.insertLiteralRows(literalRows); + } + + /** Rows for the `literals` side table (see schema.sql) — one per captured literal. */ + private collectLiteralRows(node: Node, rows: unknown[][]): void { + for (const value of node.literals ?? []) rows.push([value, node.id, node.filePath]); + } + + private insertLiteralRows(rows: unknown[][]): void { + this.runBatched( + 'insertLiterals', + 'INSERT OR IGNORE INTO literals (value, node_id, file_path) VALUES ', + '(?,?,?)', + rows + ); + } + + /** + * Node ids whose body holds one of `values` verbatim, joined to nodes so a + * row left behind by a stale index is never surfaced. Explore seeds on these + * ahead of every name-derived candidate. + */ + findNodeIdsByLiteral(values: string[], limit = 24): string[] { + if (values.length === 0) return []; + const placeholders = values.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT l.node_id AS id FROM literals l + JOIN nodes n ON n.id = l.node_id + WHERE l.value IN (${placeholders}) + ORDER BY n.kind = 'file', n.file_path, n.start_line + LIMIT ?` + ) + .all(...values, limit) as Array<{ id: string }>; + return rows.map((r) => r.id); } /** Which node kinds contribute their name to the segment vocabulary — the @@ -507,6 +545,7 @@ export class QueryBuilder { // per-.run() call overhead dominates the store phase on full indexes. const rows: unknown[][] = []; const segmentRows: unknown[][] = []; + const literalRows: unknown[][] = []; for (const node of nodes) { if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node with missing required fields:', { @@ -543,6 +582,7 @@ export class QueryBuilder { node.updatedAt ?? Date.now(), ]); if (this.isSegmentableKind(node.kind)) this.collectNameSegmentRows(node.name, segmentRows); + this.collectLiteralRows(node, literalRows); } this.runBatched( 'insertNodes', @@ -562,6 +602,7 @@ export class QueryBuilder { '(?,?)', segmentRows ); + this.insertLiteralRows(literalRows); })(); } @@ -720,9 +761,28 @@ export class QueryBuilder { this.nodeCache.delete(id); } } + if (!this.stmts.deleteLiteralsByFile) { + this.stmts.deleteLiteralsByFile = this.db.prepare('DELETE FROM literals WHERE file_path = ?'); + } + this.stmts.deleteLiteralsByFile.run(filePath); this.stmts.deleteNodesByFile.run(filePath); } + /** Refresh extracted literals even when unchanged source keeps its existing nodes. */ + replaceLiteralsForFile(filePath: string, nodes: Node[]): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM literals WHERE file_path = ?').run(filePath); + const rows: unknown[][] = []; + for (const node of nodes) this.collectLiteralRows(node, rows); + this.insertLiteralRows(rows); + })(); + } + + /** Full indexing repopulates present files and removes literals from deleted files. */ + clearLiterals(): void { + this.db.exec('DELETE FROM literals'); + } + // =========================================================================== // Name-segment vocabulary (prompt-hook graph-derived gate) // =========================================================================== @@ -3678,6 +3738,7 @@ export class QueryBuilder { this.db.transaction(() => { this.db.exec('DELETE FROM unresolved_refs'); this.db.exec('DELETE FROM edges'); + this.db.exec('DELETE FROM literals'); this.db.exec('DELETE FROM nodes'); this.db.exec('DELETE FROM files'); })(); diff --git a/src/db/schema.sql b/src/db/schema.sql index 6237d2d64..f298d3354 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -161,6 +161,22 @@ CREATE TABLE IF NOT EXISTS name_segment_vocab ( PRIMARY KEY (segment, name) ) WITHOUT ROWID; +-- Identifier-like string literal → the symbol whose body holds it +-- (extraction/literal-capture.ts). Lets an explore query that names a storage +-- key, CLI flag, or event name seed on its readers and writers: nodes_fts +-- covers names, docstrings and signatures, never string contents. Written on +-- the node write path; a file's rows are deleted with its nodes and a full +-- index clears the table. Reads still join nodes, so a stale row is never +-- surfaced. Empty on migrated databases until the next full index (the +-- source strings are not recoverable from the graph). +CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + -- Edge indexes. -- idx_edges_source / idx_edges_target are intentionally omitted — -- the (source, kind) and (target, kind) composites below cover the diff --git a/src/extraction/extraction-version.ts b/src/extraction/extraction-version.ts index 8ddab13b8..3691e95f0 100644 --- a/src/extraction/extraction-version.ts +++ b/src/extraction/extraction-version.ts @@ -21,4 +21,4 @@ * turns the re-index hint into noise — keep it honest (see CLAUDE.md, "Honesty * in the product is load-bearing"). */ -export const EXTRACTION_VERSION = 26; +export const EXTRACTION_VERSION = 27; diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 8095ee5b5..111afde27 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -2634,6 +2634,7 @@ export class ExtractionOrchestrator { existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0; const incomingHasContent = result.nodes.length > 0; if (!existingIsMarker || !incomingHasContent) { + this.queries.replaceLiteralsForFile(filePath, result.nodes); return; // No changes } } diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index c4337a091..b335a47a4 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -18,6 +18,7 @@ import type { ExtractionResult, Language } from '../../types'; import { EXTRACTORS } from '../languages'; import { getKernel, kernelSupports } from './loader'; import { decodeExtractBuffers } from './decode'; +import { captureLiterals } from '../literal-capture'; import { KERNEL_ABI_VERSION as LAYOUT_ABI, META as LAYOUT_META, @@ -233,7 +234,7 @@ export function tryKernelExtractRaw( buffers.arena.toString('utf8', errorsOff, errorsOff + errorsLen) ) as ExtractionResult['errors']; } - return { buffers, counts, errors }; + return { buffers: { ...buffers, literalSource: pre }, counts, errors }; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.includes('defer:')) { @@ -268,6 +269,7 @@ export function materializeKernelResult( filePath, language ); + if (b.literalSource !== undefined) captureLiterals(b.literalSource, decoded.nodes); decoded.durationMs = result.durationMs; return decoded; } @@ -294,6 +296,16 @@ export function tryKernelExtract( const buffers = kernel.extractFile(filePath, pre, language); const result = decodeExtractBuffers(buffers, filePath, language); POST_PASSES[language]?.(result, source); + // Literal seeds are a pass over source text and the node list, never over + // the tree (literal-capture.ts), so they need no Rust mirror — the same + // pass the wasm extractor runs at the end of extract() applies to the + // kernel's nodes here, and the two paths stay node-for-node identical. + // `pre`, not `source`: the wasm extractor captures from its own preParsed + // text, and a preParse blanks bytes a literal could otherwise be read from. + // Without this every routed language loses its seeds while markdown and the + // unrouted ones keep theirs, and a quoted-key explore query silently stops + // finding holders. + captureLiterals(pre, result.nodes); result.durationMs = Date.now() - t0; return result; } catch (err) { diff --git a/src/extraction/literal-capture.ts b/src/extraction/literal-capture.ts new file mode 100644 index 000000000..3144a1d13 --- /dev/null +++ b/src/extraction/literal-capture.ts @@ -0,0 +1,99 @@ +/** + * Identifier-like string literals — storage keys, CLI flags, event names, + * config paths — attributed to the symbol whose body contains them, so an + * explore query naming the literal seeds on its readers and writers instead + * of degrading to bag-of-words FTS (the literal is never a symbol name). + * + * The capture is a regex over the file's source text, not a tree walk: the + * per-node JS↔WASM crossing is the parse floor (docs/design/native-extraction-kernel.md), + * and a second walk would pay it again for strings alone. Quotes inside a + * comment can produce a spurious literal; the predicate below keeps only + * identifier-shaped values, so prose never qualifies. + */ +import type { Node } from '../types'; + +/** Distinct literals kept per node; a table of keys beyond this is data, not a seam. */ +const MAX_LITERALS_PER_NODE = 32; + +/** + * A value qualifies when it looks like an identifier an agent would quote + * back verbatim: leading letter or underscore (or a `-`/`--` flag prefix), + * only identifier, path, and namespace characters, and at least one + * separator — `bompus_draft_state`, `--start`, `draft.pick`, `api/v1/users`. + * A plain word (`ready`, `Error`) has no separator and stays out: it would + * seed on every function that logs it. + */ +export function isSeedLiteral(value: string): boolean { + return value.length <= 200 && /^-{0,2}[A-Za-z_][A-Za-z0-9_.:/-]{3,}$/.test(value) && /[_.:/-]/.test(value); +} + +/** + * Quoted spans in the query text, plus any bare whitespace-delimited run that + * qualifies. Deduplicated, query order. + */ +export function seedLiteralsInQuery(query: string): string[] { + const out = new Set(); + for (const m of query.matchAll(/["'`]([^"'`\s]+)["'`]/g)) { + const quoted = m[1] ?? ''; + if (isSeedLiteral(quoted)) out.add(quoted); + } + for (const run of query.split(/\s+/)) { + const bare = run.replace(/^[("'`[]+|[)"'`\],.;:?!]+$/g, ''); + if (isSeedLiteral(bare)) out.add(bare); + } + return [...out]; +} + +/** Single-line quoted strings; template literals with `${` interpolation are skipped. */ +const STRING_RE = /(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g; + +/** + * Attach qualifying literals in `source` to `nodes` (mutated): each literal + * goes to the innermost non-file node whose source range contains it, else + * the file node. Both extraction paths expose UTF-16 columns (the native + * kernel converts its tree-sitter byte columns before emitting nodes). + */ +export function captureLiterals(source: string, nodes: Node[]): void { + if (nodes.length === 0 || source.length === 0) return; + const symbols = nodes.filter((n) => n.kind !== 'file' && n.kind !== 'import'); + const fileNode = nodes.find((n) => n.kind === 'file'); + const lineStarts = [0]; + for (let i = 0; i < source.length; i++) if (source.charCodeAt(i) === 10) lineStarts.push(i + 1); + const lineOf = (offset: number): number => { + let lo = 0; + let hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if ((lineStarts[mid] ?? 0) <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; + }; + const seen = new Map>(); + for (const m of source.matchAll(STRING_RE)) { + const value = m[2] ?? ''; + if (m[1] === '`' && value.includes('${')) continue; + if (!isSeedLiteral(value)) continue; + const line = lineOf(m.index ?? 0); + const startColumn = (m.index ?? 0) - (lineStarts[line - 1] ?? 0); + const endColumn = startColumn + m[0].length; + let owner: Node | undefined; + for (const n of symbols) { + if (n.startLine > line || n.endLine < line) continue; + if (n.startLine === line && n.startColumn > startColumn) continue; + if (n.endLine === line && n.endColumn < endColumn) continue; + // A contained range is deeper; on identical ranges the later walker node wins. + if (!owner || (n.startLine > owner.startLine || + (n.startLine === owner.startLine && n.startColumn >= owner.startColumn)) && + (n.endLine < owner.endLine || + (n.endLine === owner.endLine && n.endColumn <= owner.endColumn))) owner = n; + } + owner ??= fileNode; + if (!owner) continue; + let set = seen.get(owner.id); + if (!set) seen.set(owner.id, (set = new Set())); + if (set.size >= MAX_LITERALS_PER_NODE || set.has(value)) continue; + set.add(value); + (owner.literals ??= []).push(value); + } +} diff --git a/src/extraction/store-worker.ts b/src/extraction/store-worker.ts index 6493e0040..bbc00a20a 100644 --- a/src/extraction/store-worker.ts +++ b/src/extraction/store-worker.ts @@ -32,6 +32,7 @@ import { QueryBuilder } from '../db/queries'; import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter'; import { finalizeStoreBundle, type KernelStoreBundle, type StoreBundle } from './store-writer'; import { decodeExtractBuffers } from './kernel/decode'; +import { captureLiterals } from './literal-capture'; if (!parentPort) { throw new Error('store-worker must be run as a worker thread'); @@ -70,6 +71,9 @@ function decodeKernelBundle(bundle: KernelStoreBundle): StoreBundle { bundle.filePath, bundle.language ); + if (bundle.buffers.literalSource !== undefined) { + captureLiterals(bundle.buffers.literalSource, decoded.nodes); + } return finalizeStoreBundle(decoded, bundle.filePath, bundle.language, bundle.file); } diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index 20f5dbb26..2b3418a8c 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -32,6 +32,7 @@ import { VueExtractor } from './vue-extractor'; import { MyBatisExtractor } from './mybatis-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse } from './kernel'; +import { captureLiterals } from './literal-capture'; import { getAllFrameworkResolvers, getApplicableFrameworks, @@ -581,6 +582,7 @@ export class TreeSitterExtractor { // nodes and import refs are complete and the file node is still pushed. this.flushFnRefCandidates(); this.flushValueRefs(); + captureLiterals(this.source, this.nodes); if (packageNodeId) this.nodeStack.pop(); this.nodeStack.pop(); diff --git a/src/index.ts b/src/index.ts index 9941b9c16..c4808dc01 100644 --- a/src/index.ts +++ b/src/index.ts @@ -58,6 +58,7 @@ import ignore from 'ignore'; import { loadDeprioritizePatterns } from './project-config'; import { CodeGraphPackageVersion } from './mcp/version'; import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; +import { seedLiteralsInQuery } from './extraction/literal-capture'; import { createYielder } from './resolution/cooperative-yield'; import { minRefsForPool } from './resolution/resolver-pool'; @@ -527,6 +528,7 @@ export class CodeGraph { // path as every file (re-)indexes below — so a full index is also the // orphan-cleanup pass for names deleted since the last one. try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ } + try { this.queries.clearLiterals(); } catch { /* literals are repopulated even for unchanged files */ } // Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync // triggers, rebuild nodes_fts once from the nodes table afterwards. // Crash inside the window is healed on the next DatabaseConnection.open. @@ -2146,7 +2148,24 @@ export class CodeGraph { seedNames = []; } } - return this.contextBuilder.findRelevantContext(query, { ...options, seedNames }); + const seedNodeIds = options?.seedNodeIds ?? this.findLiteralSeedIds(query); + return this.contextBuilder.findRelevantContext(query, { ...options, seedNames, seedNodeIds }); + } + + /** + * Literal seeds: a storage key, flag, or event name quoted in the query is + * never a symbol name, so resolve it through the literals table to the + * symbols whose bodies hold it. `CODEGRAPH_LITERAL_SEEDS=0` is the ablation + * switch; failures (pre-v10 database) degrade to no seeds. Explore's file + * sort calls this too, so a holder file ranks as a named file. + */ + findLiteralSeedIds(query: string): string[] { + if (process.env.CODEGRAPH_LITERAL_SEEDS === '0') return []; + try { + return this.queries.findNodeIdsByLiteral(seedLiteralsInQuery(query)); + } catch { + return []; + } } /** diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index 4cb22150d..2b34b9e47 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -51,6 +51,8 @@ calls; a grep/read exploration is dozens. ## How to query +Storage keys, command-line flags, and event names in a query also match symbols containing those exact string literals. Existing indexes need a full rebuild to capture them. + - **Almost any question — "how does X work", architecture, a bug, "what/where is X", or surveying an area** → \`codegraph_explore\` with a natural-language question or the relevant names. ONE capped call returns the verbatim source grouped by file; most often the ONLY call you need. - **"How does X reach/become Y? / the flow / the path from X to Y"** → \`codegraph_explore\`, naming the symbols that span the flow (e.g. \`mutateElement renderScene\`) — it surfaces the call path among them, riding dynamic-dispatch hops, and returns their source. - **Reading or editing a file/symbol you can name** → put its name or file path in the \`codegraph_explore\` query — it returns that current line-numbered source (safe to \`Edit\` from) with the call path and blast radius attached, so you don't Read it separately. For an overloaded name it returns every matching definition's body in one call. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 42fdd2288..930734d14 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -3300,7 +3300,7 @@ export class ToolHandler { } catch { budget = getExploreOutputBudget(Infinity); } - const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + let maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); // File paths named in the query become PINNED files: guaranteed admission, // top of the rank order, funded first — and their span is REMOVED from the @@ -3328,6 +3328,19 @@ export class ToolHandler { } catch { /* path pinning must never fail an explore call */ } } const pinnedSet = new Set(pinnedFiles); + // A literal quoted in the query names every file holding it, so the + // default file cap (sized for ranked padding) rises to the holder count; + // the character budget still bounds the answer, and an explicit maxFiles + // stands. + const literalSeedIds = cg.findLiteralSeedIds(matchQuery); + if (!args.maxFiles && literalSeedIds.length > 0) { + const holderFiles = new Set(); + for (const id of literalSeedIds) { + const n = cg.getNode(id); + if (n) holderFiles.add(n.filePath); + } + maxFiles = clamp(Math.max(maxFiles, holderFiles.size), 1, 12); + } const pinnedOrder = new Map(pinnedFiles.map((p, i) => [p, i])); // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG @@ -3678,6 +3691,13 @@ export class ToolHandler { } } } + // Exact literal holders deserve the same source priority as named symbols. + for (const id of literalSeedIds) { + if (subgraph.nodes.has(id)) { + namedSeedIds.add(id); + tierSeedIds.add(id); + } + } // Step 2: Group nodes by file, score by relevance // `peripheral` accumulates separately so it can be capped — see diff --git a/src/types.ts b/src/types.ts index 44ffaf4e4..e9a412775 100644 --- a/src/types.ts +++ b/src/types.ts @@ -188,6 +188,13 @@ export interface Node { /** Generic type parameters */ typeParameters?: string[]; + /** + * Identifier-like string literals inside this symbol's body (storage keys, + * CLI flags, event names — see extraction/literal-capture.ts). Stored in the + * `literals` side table, not a nodes column; seeds explore on an exact hit. + */ + literals?: string[]; + /** * Normalized return/result type name for a function/method (the bare class * name, smart-pointer pointee unwrapped). Captured for C/C++ so resolution @@ -302,6 +309,8 @@ export interface ExtractionResult { edges: Uint8Array; refs: Uint8Array; arena: Uint8Array; + /** Preparsed source for literal attribution after deferred node decoding. */ + literalSource?: string; }; kernelCounts?: { nodes: number; edges: number; refs: number }; } @@ -698,4 +707,12 @@ export interface FindRelevantContextOptions { * SEGMENTS the query's words name are seeded here instead. */ seedNames?: string[]; + + /** + * Node ids whose body holds a string literal the query quoted verbatim + * (`literals` table, CodeGraph.findRelevantContext). Ranked above every + * name-derived candidate: an exact literal is the strongest evidence a + * query carries, and the symbol is never named after it. + */ + seedNodeIds?: string[]; }