From 91544d26ac83b3c034ed7065a967e8d620eb0024 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 14 Aug 2026 03:45:01 +0000 Subject: [PATCH 1/5] =?UTF-8?q?feat(run-log):=20add=20@agentic-kit/run-log?= =?UTF-8?q?=20=E2=80=94=20append-only=20agent=20run=20log=20with=20pi=20en?= =?UTF-8?q?tries=20stored=20verbatim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/run-tests.yaml | 2 +- agentic/run-log/README.md | 95 ++++++ .../run-log/__tests__/entry-points.test.ts | 40 +++ agentic/run-log/__tests__/file-store.test.ts | 86 ++++++ agentic/run-log/__tests__/fixtures.ts | 112 +++++++ agentic/run-log/__tests__/projectors.test.ts | 287 ++++++++++++++++++ agentic/run-log/__tests__/record.test.ts | 102 +++++++ agentic/run-log/__tests__/store.test.ts | 135 ++++++++ agentic/run-log/jest.config.js | 21 ++ agentic/run-log/package.json | 38 +++ agentic/run-log/src/file-store.ts | 123 ++++++++ agentic/run-log/src/follow.ts | 97 ++++++ agentic/run-log/src/index.ts | 106 +++++++ agentic/run-log/src/pi-entry.ts | 238 +++++++++++++++ agentic/run-log/src/projectors/parts.ts | 234 ++++++++++++++ agentic/run-log/src/projectors/session.ts | 94 ++++++ agentic/run-log/src/projectors/tool-state.ts | 192 ++++++++++++ agentic/run-log/src/projectors/usage.ts | 113 +++++++ agentic/run-log/src/record.ts | 115 +++++++ agentic/run-log/src/store.ts | 103 +++++++ agentic/run-log/tsconfig.esm.json | 7 + agentic/run-log/tsconfig.json | 8 + pnpm-lock.yaml | 3 + 23 files changed, 2350 insertions(+), 1 deletion(-) create mode 100644 agentic/run-log/README.md create mode 100644 agentic/run-log/__tests__/entry-points.test.ts create mode 100644 agentic/run-log/__tests__/file-store.test.ts create mode 100644 agentic/run-log/__tests__/fixtures.ts create mode 100644 agentic/run-log/__tests__/projectors.test.ts create mode 100644 agentic/run-log/__tests__/record.test.ts create mode 100644 agentic/run-log/__tests__/store.test.ts create mode 100644 agentic/run-log/jest.config.js create mode 100644 agentic/run-log/package.json create mode 100644 agentic/run-log/src/file-store.ts create mode 100644 agentic/run-log/src/follow.ts create mode 100644 agentic/run-log/src/index.ts create mode 100644 agentic/run-log/src/pi-entry.ts create mode 100644 agentic/run-log/src/projectors/parts.ts create mode 100644 agentic/run-log/src/projectors/session.ts create mode 100644 agentic/run-log/src/projectors/tool-state.ts create mode 100644 agentic/run-log/src/projectors/usage.ts create mode 100644 agentic/run-log/src/record.ts create mode 100644 agentic/run-log/src/store.ts create mode 100644 agentic/run-log/tsconfig.esm.json create mode 100644 agentic/run-log/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index b19278dea9..b76174b9b0 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/run-log/README.md b/agentic/run-log/README.md new file mode 100644 index 0000000000..34fd6cebd2 --- /dev/null +++ b/agentic/run-log/README.md @@ -0,0 +1,95 @@ +

+ +

+ +# @agentic-kit/run-log + +The append-only **run log**: one ordered record of what an agent run did, wherever it ran. + +A coding agent run has to be observable from a desktop app, a web UI and a CLI; it has to be resumable after a crash, a restart, or a move from the cloud to a laptop; and it has to be meterable. Those are usually four subsystems. Here they are four *projections* of one log: + +``` +pi session entries ──► run log (append-only) ──┬──► transcript parts (what a UI draws) + ├──► usage totals (what a run cost) + ├──► tool + approval (what is blocked) + └──► pi session file (how it resumes) +``` + +The log stores **pi entries verbatim** under four platform-owned fields. pi already versions and migrates its own session format, so re-encoding it into a second semantic event model would mean maintaining a translation layer that silently loses whatever pi adds next. + +```ts +{ + runId, // which run + seq, // 1-based, gapless, strictly increasing + recordedAt, // when the platform durably recorded it + piSessionVersion, // pi's session format version at write time + entry // the pi entry, byte-for-byte +} +``` + +## Install + +```sh +npm install @agentic-kit/run-log +``` + +## Writing + +Writing is an `append` against a store. Appends are **idempotent by pi entry id**, so a writer that restarts and replays its tail does not duplicate history. + +```ts +import { MemoryRunLogStore } from '@agentic-kit/run-log'; + +const store = new MemoryRunLogStore(); +await store.append('run-1', [entry]); // → the records actually written +await store.append('run-1', [entry]); // → [] — already present +``` + +Two stores ship here: `MemoryRunLogStore` (the reference implementation, and the test double) and `FileRunLogStore` from the node-only entry point: + +```ts +import { FileRunLogStore, writeSessionFile } from '@agentic-kit/run-log/file-store'; +``` + +The main entry is **browser-safe** — renderers import the projectors, so nothing in it may reach for a node builtin. Anything touching the filesystem lives behind `/file-store` (the same split, for the same reason, as `12factor-env/dotenv`). + +The durable store is Postgres, in `constructive-db`: `append` becomes an insert, `read` a keyset scan. Neither is in this package, because the interface is the contract. + +## Reading + +Every surface is the same reader: hold a cursor, ask for what came after it. That is what makes a local run and a cloud run indistinguishable to a UI. + +```ts +import { follow, projectParts, readAll } from '@agentic-kit/run-log'; + +const records = await readAll(store, 'run-1'); + +for await (const batch of follow(store, 'run-1', { waitForChange })) { + render(projectParts(batch).parts); +} +``` + +`waitForChange` (Postgres `LISTEN/NOTIFY`, IPC, a websocket) is an *optimisation*: it races the poll delay, and without one `follow` degrades to polling. No new transport is required for a run to stream. + +## Projections + +| Projection | Answers | +| --- | --- | +| `projectParts` | the renderable transcript — a tool call and its later result collapse into one part, so no renderer correlates messages itself | +| `projectUsage` | tokens and cost, per `provider/model` and per run, including nested tool usage and compaction calls | +| `projectToolState` | what is running, and what is waiting on a human (approvals ride in the log as pi `custom` messages, so a surface that reconnects hours later still sees a pending request) | +| `projectSession` | a pi session file — the resume path, cloud → local included | + +All four are pure and total: the same records always produce the same output, whichever host wrote them, which is the property the tests assert as *placement invariance*. + +Unreadable input fails loudly. A corrupt record, a mixed-version log, or a session whose header is not first throws rather than degrading into an empty transcript — a run that silently renders as blank is worse than one that reports it cannot be read. An *unknown* entry type is different: it is carried through as an `unknown` part, so a log written by a newer pi still renders. + +## Related + +- [`@agentic-kit/pi`](../pi) — Constructive's typed db tools as a pi extension +- [`@agentic-kit/harness`](../harness) — host-neutral gates and policy +- [`agentic-server`](../agentic-server) — the metered inference gateway + +## License + +SEE LICENSE IN LICENSE diff --git a/agentic/run-log/__tests__/entry-points.test.ts b/agentic/run-log/__tests__/entry-points.test.ts new file mode 100644 index 0000000000..ca66972f94 --- /dev/null +++ b/agentic/run-log/__tests__/entry-points.test.ts @@ -0,0 +1,40 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * The main entry is imported by browsers, Electron renderers and Next client + * components. A `node:` import anywhere in its module graph breaks those + * bundles, and it breaks them at build time in someone else's repo — so the + * boundary is asserted here rather than discovered there. + */ +const srcDir = join(__dirname, '..', 'src'); + +const sources = (dir: string): string[] => + readdirSync(dir).flatMap((name) => { + const path = join(dir, name); + if (statSync(path).isDirectory()) return sources(path); + return name.endsWith('.ts') ? [path] : []; + }); + +/** Only this file may touch the filesystem. */ +const NODE_ONLY = ['file-store.ts']; + +describe('entry points', () => { + const browserSafe = sources(srcDir).filter((path) => !NODE_ONLY.some((name) => path.endsWith(name))); + + it.each(browserSafe.map((path) => [path.slice(srcDir.length + 1), path]))( + 'src/%s imports no node builtin', + (_name, path) => { + const contents = readFileSync(path, 'utf8'); + expect(contents).not.toMatch(/from ['"]node:/); + expect(contents).not.toMatch(/require\(['"]node:/); + } + ); + + it('keeps the filesystem store out of the main entry', () => { + const index = readFileSync(join(srcDir, 'index.ts'), 'utf8'); + expect(index).not.toMatch(/from ['"]\.\/file-store['"]/); + // eslint-disable-next-line @typescript-eslint/no-var-requires + expect(Object.keys(require('../src'))).not.toContain('FileRunLogStore'); + }); +}); diff --git a/agentic/run-log/__tests__/file-store.test.ts b/agentic/run-log/__tests__/file-store.test.ts new file mode 100644 index 0000000000..34e8b593d0 --- /dev/null +++ b/agentic/run-log/__tests__/file-store.test.ts @@ -0,0 +1,86 @@ +import { appendFileSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { parseSessionJsonl, projectParts, readAll } from '../src'; +import { FileRunLogStore, writeSessionFile } from '../src/file-store'; +import { assistantText, assistantToolCall, header, resetIds, toolResult, userMessage } from './fixtures'; + +let dir: string; +let path: string; + +beforeEach(() => { + resetIds(); + dir = mkdtempSync(join(tmpdir(), 'run-log-')); + path = join(dir, 'nested', 'run.jsonl'); +}); + +describe('FileRunLogStore', () => { + it('creates the log on first append and survives a new store instance', async () => { + const writer = new FileRunLogStore({ path }); + await writer.append('run-1', [header(), userMessage('hi')]); + await writer.append('run-1', [assistantText('hello')]); + + const reader = new FileRunLogStore({ path }); + const records = await readAll(reader, 'run-1'); + expect(records.map((r) => r.seq)).toEqual([1, 2, 3]); + expect(readFileSync(path, 'utf8').trimEnd().split('\n')).toHaveLength(3); + }); + + it('reads nothing for a log that does not exist yet', async () => { + expect((await new FileRunLogStore({ path }).read('run-1')).records).toEqual([]); + }); + + it('resumes sequence numbering after a process restart', async () => { + await new FileRunLogStore({ path }).append('run-1', [userMessage('a')]); + const written = await new FileRunLogStore({ path }).append('run-1', [userMessage('b', 2)]); + expect(written[0].seq).toBe(2); + }); + + it('skips entries already on disk, so a restarted writer can replay its tail', async () => { + const entries = [header(), userMessage('a'), assistantText('b')]; + await new FileRunLogStore({ path }).append('run-1', entries); + const retry = await new FileRunLogStore({ path }).append('run-1', entries); + expect(retry).toEqual([]); + expect(readFileSync(path, 'utf8').trimEnd().split('\n')).toHaveLength(3); + }); + + it('keeps runs separate within one file', async () => { + const store = new FileRunLogStore({ path }); + await store.append('run-1', [userMessage('a')]); + await store.append('run-2', [userMessage('b')]); + expect((await store.read('run-2')).records[0].seq).toBe(1); + expect((await store.read('run-1')).records).toHaveLength(1); + }); + + it('throws on a truncated or corrupt line instead of returning a partial log', async () => { + const store = new FileRunLogStore({ path }); + await store.append('run-1', [userMessage('a')]); + appendFileSync(path, '{"runId":"run-1","seq":2,'); + await expect(store.read('run-1')).rejects.toThrow(/line 2 is not valid JSON/); + + writeFileSync(path, '{"runId":"run-1","seq":1,"recordedAt":"x","piSessionVersion":3}\n'); + await expect(store.read('run-1')).rejects.toThrow(/must be an object/); + }); +}); + +describe('writeSessionFile', () => { + it('writes a pi session file the log can be resumed from', async () => { + const store = new FileRunLogStore({ path }); + await store.append('run-1', [ + header(), + userMessage('add a test'), + assistantToolCall({ id: 'call-1', name: 'write_file' }), + toolResult({ toolCallId: 'call-1', toolName: 'write_file', text: 'ok' }) + ]); + const records = await readAll(store, 'run-1'); + + const sessionPath = writeSessionFile(join(dir, 'sessions', 'session-1.jsonl'), records); + const entries = parseSessionJsonl(readFileSync(sessionPath, 'utf8')); + + expect(entries[0]).toMatchObject({ type: 'session', id: 'session-1' }); + expect(entries).toHaveLength(4); + // The transcript a UI would draw is unchanged by the round trip. + expect(projectParts(records).parts).toHaveLength(2); + }); +}); diff --git a/agentic/run-log/__tests__/fixtures.ts b/agentic/run-log/__tests__/fixtures.ts new file mode 100644 index 0000000000..b8cf8d3224 --- /dev/null +++ b/agentic/run-log/__tests__/fixtures.ts @@ -0,0 +1,112 @@ +import type { PiSessionEntry, PiSessionHeader, PiUsage } from '../src/pi-entry'; + +let counter = 0; +const nextId = (): string => { + counter += 1; + return counter.toString(16).padStart(8, '0'); +}; + +export const resetIds = (): void => { + counter = 0; +}; + +const at = (n: number): string => new Date(Date.UTC(2026, 0, 1, 0, 0, n)).toISOString(); + +export const usage = (over: Partial = {}): PiUsage => ({ + input: 100, + output: 20, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 120, + cost: { input: 0.001, output: 0.002, cacheRead: 0, cacheWrite: 0, total: 0.003 }, + ...over +}); + +export const header = (over: Partial = {}): PiSessionEntry => ({ + type: 'session', + version: 3, + id: 'session-1', + timestamp: at(0), + cwd: '/repo', + ...over +}); + +const entry = (type: string, rest: Record, seq: number): PiSessionEntry => + ({ type, id: nextId(), parentId: null, timestamp: at(seq), ...rest }) as PiSessionEntry; + +export const userMessage = (text: string, seq = 1): PiSessionEntry => + entry('message', { message: { role: 'user', content: text, timestamp: seq } }, seq); + +export const assistantText = (text: string, seq = 2, over: Record = {}): PiSessionEntry => + entry( + 'message', + { + message: { + role: 'assistant', + content: [{ type: 'text', text }], + provider: 'anthropic', + model: 'claude-sonnet-4-5', + usage: usage(), + stopReason: 'stop', + ...over + } + }, + seq + ); + +export const assistantToolCall = ( + call: { id: string; name: string; arguments?: Record }, + seq = 3, + over: Record = {} +): PiSessionEntry => + entry( + 'message', + { + message: { + role: 'assistant', + content: [{ type: 'toolCall', ...call }], + provider: 'anthropic', + model: 'claude-sonnet-4-5', + usage: usage({ output: 30, totalTokens: 130 }), + stopReason: 'toolUse', + ...over + } + }, + seq + ); + +export const toolResult = ( + result: { toolCallId: string; toolName: string; text: string; isError?: boolean; usage?: PiUsage }, + seq = 4 +): PiSessionEntry => + entry( + 'message', + { + message: { + role: 'toolResult', + toolCallId: result.toolCallId, + toolName: result.toolName, + content: [{ type: 'text', text: result.text }], + isError: result.isError ?? false, + ...(result.usage ? { usage: result.usage } : {}) + } + }, + seq + ); + +export const custom = ( + message: { customType: string; content: string; details?: unknown; display?: boolean }, + seq = 5 +): PiSessionEntry => entry('message', { message: { role: 'custom', display: true, ...message } }, seq); + +export const bash = (command: string, output: string, exitCode = 0, seq = 6): PiSessionEntry => + entry('message', { message: { role: 'bashExecution', command, output, exitCode } }, seq); + +export const compaction = (summary: string, seq = 7, over: Record = {}): PiSessionEntry => + entry('compaction', { summary, tokensBefore: 50_000, ...over }, seq); + +export const branchSummary = (summary: string, seq = 8, over: Record = {}): PiSessionEntry => + entry('branch_summary', { summary, fromId: 'aaaaaaaa', ...over }, seq); + +/** An entry type this version of the package has never seen. */ +export const futureEntry = (seq = 9): PiSessionEntry => entry('quantum_thought', { intensity: 11 }, seq); diff --git a/agentic/run-log/__tests__/projectors.test.ts b/agentic/run-log/__tests__/projectors.test.ts new file mode 100644 index 0000000000..e435d18901 --- /dev/null +++ b/agentic/run-log/__tests__/projectors.test.ts @@ -0,0 +1,287 @@ +import { + APPROVAL_REQUEST_TYPE, + APPROVAL_RESOLUTION_TYPE, + approvalRequestMessage, + approvalResolutionMessage, + MemoryRunLogStore, + modelKey, + parseSessionJsonl, + projectParts, + projectSession, + projectToolState, + projectUsage, + readAll, + type RunEventRecord, + wrapEntry +} from '../src'; +import type { PiSessionEntry } from '../src/pi-entry'; +import { + assistantText, + assistantToolCall, + bash, + branchSummary, + compaction, + custom, + futureEntry, + header, + resetIds, + toolResult, + usage, + userMessage +} from './fixtures'; + +beforeEach(resetIds); + +const recordsOf = (...entries: PiSessionEntry[]): RunEventRecord[] => + entries.map((entry, i) => wrapEntry({ runId: 'run-1', seq: i + 1, entry, recordedAt: `2026-01-01T00:00:0${String(i)}.000Z` })); + +describe('projectParts', () => { + it('renders a full turn: user text, assistant text, tool call collapsed with its result', () => { + const { parts, sessionId, cwd } = projectParts( + recordsOf( + header(), + userMessage('add a test'), + assistantText('on it'), + assistantToolCall({ id: 'call-1', name: 'write_file', arguments: { path: 'a.ts' } }), + toolResult({ toolCallId: 'call-1', toolName: 'write_file', text: 'wrote a.ts' }) + ) + ); + + expect(sessionId).toBe('session-1'); + expect(cwd).toBe('/repo'); + expect(parts).toEqual([ + { kind: 'text', role: 'user', text: 'add a test', seq: 2, entryId: expect.any(String) }, + { + kind: 'text', + role: 'assistant', + text: 'on it', + model: 'claude-sonnet-4-5', + provider: 'anthropic', + seq: 3, + entryId: expect.any(String) + }, + { + kind: 'tool', + toolCallId: 'call-1', + name: 'write_file', + arguments: { path: 'a.ts' }, + status: 'completed', + output: 'wrote a.ts', + settledSeq: 5, + seq: 4, + entryId: expect.any(String) + } + ]); + }); + + it('leaves an unsettled tool call in the requested state', () => { + const [, tool] = projectParts( + recordsOf(userMessage('go'), assistantToolCall({ id: 'call-1', name: 'bash' })) + ).parts; + expect(tool).toMatchObject({ kind: 'tool', status: 'requested' }); + expect((tool as { output?: string }).output).toBeUndefined(); + }); + + it('marks an errored tool result as failed', () => { + const { parts } = projectParts( + recordsOf( + assistantToolCall({ id: 'call-1', name: 'bash' }), + toolResult({ toolCallId: 'call-1', toolName: 'bash', text: 'boom', isError: true }) + ) + ); + expect(parts[0]).toMatchObject({ status: 'failed', output: 'boom' }); + }); + + it('keeps a tool result whose call is outside the read window', () => { + const { parts } = projectParts(recordsOf(toolResult({ toolCallId: 'call-9', toolName: 'bash', text: 'ok' }))); + expect(parts).toEqual([ + expect.objectContaining({ kind: 'tool', toolCallId: 'call-9', status: 'completed', arguments: {} }) + ]); + }); + + it('projects thinking, bash, custom and summary entries', () => { + const { parts } = projectParts( + recordsOf( + assistantText('answer', 2, { content: [{ type: 'thinking', thinking: 'hmm' }, { type: 'text', text: 'answer' }] }), + bash('ls', 'a.ts\n'), + custom({ customType: 'constructive.note', content: 'heads up' }), + compaction('summary so far'), + branchSummary('branched') + ) + ); + + expect(parts.map((p) => p.kind)).toEqual(['thinking', 'text', 'bash', 'custom', 'summary', 'summary']); + expect(parts[2]).toMatchObject({ kind: 'bash', command: 'ls', output: 'a.ts\n', exitCode: 0 }); + expect(parts[3]).toMatchObject({ customType: 'constructive.note', text: 'heads up', display: true }); + expect(parts[4]).toMatchObject({ reason: 'compaction', summary: 'summary so far' }); + expect(parts[5]).toMatchObject({ reason: 'branch', summary: 'branched' }); + }); + + it('surfaces an entry type it does not understand instead of dropping it', () => { + const { parts } = projectParts(recordsOf(futureEntry())); + expect(parts).toEqual([ + expect.objectContaining({ kind: 'unknown', entryType: 'quantum_thought' }) + ]); + }); +}); + +describe('projectUsage', () => { + it('totals tokens and cost per model and for the run', () => { + const totals = projectUsage( + recordsOf( + userMessage('hi'), + assistantText('a'), + assistantText('b', 3, { model: 'claude-haiku-4-5', usage: usage({ input: 10, output: 5, totalTokens: 15 }) }) + ) + ); + + expect(totals).toMatchObject({ input: 110, output: 25, totalTokens: 135, calls: 2 }); + expect(totals.cost).toBeCloseTo(0.006, 6); + expect(Object.keys(totals.byModel)).toEqual([ + modelKey('anthropic', 'claude-sonnet-4-5'), + modelKey('anthropic', 'claude-haiku-4-5') + ]); + expect(totals.byModel['anthropic/claude-haiku-4-5']).toMatchObject({ input: 10, calls: 1 }); + }); + + it('counts nested tool usage and compaction, attributed to the requesting model', () => { + const totals = projectUsage( + recordsOf( + assistantToolCall({ id: 'call-1', name: 'subagent' }), + toolResult({ toolCallId: 'call-1', toolName: 'subagent', text: 'done', usage: usage({ input: 7, output: 3, totalTokens: 10 }) }), + compaction('summary', 7, { usage: usage({ input: 1, output: 1, totalTokens: 2 }) }) + ) + ); + + expect(totals.calls).toBe(3); + expect(totals.input).toBe(108); + expect(totals.byModel['anthropic/claude-sonnet-4-5'].calls).toBe(3); + }); + + it('derives a missing total from the parts the provider did report', () => { + const totals = projectUsage( + recordsOf(assistantText('a', 2, { usage: { input: 4, output: 6, cacheRead: 2, cacheWrite: 1 } })) + ); + expect(totals.totalTokens).toBe(13); + expect(totals.cost).toBe(0); + }); + + it('is zero for a run with no model calls', () => { + expect(projectUsage(recordsOf(header(), userMessage('hi')))).toMatchObject({ totalTokens: 0, calls: 0 }); + }); +}); + +describe('projectToolState', () => { + it('tracks a tool through approval to completion', () => { + const records = recordsOf( + assistantToolCall({ id: 'call-1', name: 'deploy' }), + custom(approvalRequestMessage({ toolCallId: 'call-1', prompt: 'deploy to prod?' })), + custom({ ...approvalResolutionMessage({ toolCallId: 'call-1', decision: 'approved', actorId: 'user-1' }) }), + toolResult({ toolCallId: 'call-1', toolName: 'deploy', text: 'deployed' }) + ); + + const midway = projectToolState(records.slice(0, 2)); + expect(midway.tools['call-1'].status).toBe('awaiting-approval'); + expect(midway.pendingApprovals).toEqual([ + expect.objectContaining({ toolCallId: 'call-1', prompt: 'deploy to prod?' }) + ]); + + const approved = projectToolState(records.slice(0, 3)); + expect(approved.tools['call-1'].status).toBe('running'); + expect(approved.pendingApprovals).toEqual([]); + expect(approved.tools['call-1'].approval).toMatchObject({ decision: 'approved', actorId: 'user-1' }); + + const done = projectToolState(records); + expect(done.tools['call-1']).toMatchObject({ status: 'completed', output: 'deployed' }); + }); + + it('marks a rejected tool as rejected', () => { + const state = projectToolState( + recordsOf( + assistantToolCall({ id: 'call-1', name: 'deploy' }), + custom(approvalRequestMessage({ toolCallId: 'call-1', prompt: 'ok?' })), + custom(approvalResolutionMessage({ toolCallId: 'call-1', decision: 'rejected', reason: 'not now' })) + ) + ); + expect(state.tools['call-1']).toMatchObject({ status: 'rejected' }); + expect(state.tools['call-1'].approval).toMatchObject({ decision: 'rejected', reason: 'not now' }); + }); + + it('throws on an approval message that cannot be attributed', () => { + expect(() => + projectToolState(recordsOf(custom({ customType: APPROVAL_REQUEST_TYPE, content: 'ok?' }))) + ).toThrow(/no toolCallId/); + expect(() => + projectToolState(recordsOf(custom({ customType: APPROVAL_RESOLUTION_TYPE, content: 'yes', details: {} }))) + ).toThrow(/no toolCallId/); + }); + + it('orders pending approvals oldest first', () => { + const state = projectToolState( + recordsOf( + assistantToolCall({ id: 'call-1', name: 'a' }), + assistantToolCall({ id: 'call-2', name: 'b' }), + custom(approvalRequestMessage({ toolCallId: 'call-2', prompt: 'b?' })), + custom(approvalRequestMessage({ toolCallId: 'call-1', prompt: 'a?' })) + ) + ); + expect(state.pendingApprovals.map((a) => a.toolCallId)).toEqual(['call-2', 'call-1']); + }); +}); + +describe('projectSession', () => { + it('projects a resumable session file with the header first', () => { + const records = recordsOf(header(), userMessage('hi'), assistantText('hello')); + const { jsonl, entries, piSessionVersion } = projectSession(records); + + expect(piSessionVersion).toBe(3); + expect(entries[0]).toMatchObject({ type: 'session', id: 'session-1' }); + expect(jsonl.endsWith('\n')).toBe(true); + expect(parseSessionJsonl(jsonl)).toEqual(entries); + }); + + it('synthesises a header when the log has none', () => { + const { entries } = projectSession(recordsOf(userMessage('hi')), { sessionId: 's-9', cwd: '/w' }); + expect(entries[0]).toMatchObject({ type: 'session', id: 's-9', cwd: '/w', version: 3 }); + expect(entries).toHaveLength(2); + }); + + it('refuses to project an unloadable session', () => { + expect(() => projectSession(recordsOf(userMessage('hi'), header()))).toThrow(/requires it first/); + + const mixed = recordsOf(userMessage('hi'), userMessage('there', 2)); + mixed[1] = { ...mixed[1], piSessionVersion: 2 }; + expect(() => projectSession(mixed)).toThrow(/mixes pi session versions/); + }); + + it('rejects a malformed session file rather than returning a partial one', () => { + expect(() => parseSessionJsonl('{"type":"session"}\nnot json\n')).toThrow(/line 2 is not valid JSON/); + }); +}); + +describe('placement invariance', () => { + it('projects identically whether entries were appended in one batch or streamed', async () => { + const entries = [ + header(), + userMessage('add a test'), + assistantToolCall({ id: 'call-1', name: 'write_file', arguments: { path: 'a.ts' } }), + toolResult({ toolCallId: 'call-1', toolName: 'write_file', text: 'wrote a.ts' }), + assistantText('done', 5) + ]; + + const cloud = new MemoryRunLogStore(); + await cloud.append('run-1', entries); + + const local = new MemoryRunLogStore(); + for (const entry of entries) await local.append('run-1', [entry]); + + const cloudRecords = await readAll(cloud, 'run-1'); + const localRecords = await readAll(local, 'run-1'); + + expect(localRecords.map((r) => r.seq)).toEqual(cloudRecords.map((r) => r.seq)); + expect(projectParts(localRecords)).toEqual(projectParts(cloudRecords)); + expect(projectUsage(localRecords)).toEqual(projectUsage(cloudRecords)); + expect(projectToolState(localRecords)).toEqual(projectToolState(cloudRecords)); + expect(projectSession(localRecords).jsonl).toEqual(projectSession(cloudRecords).jsonl); + }); +}); diff --git a/agentic/run-log/__tests__/record.test.ts b/agentic/run-log/__tests__/record.test.ts new file mode 100644 index 0000000000..3085e55779 --- /dev/null +++ b/agentic/run-log/__tests__/record.test.ts @@ -0,0 +1,102 @@ +import { + assertOrdered, + assertPiSessionEntry, + assertRunEventRecord, + idempotencyKey, + RUN_LOG_WRAPPER_VERSION, + SUPPORTED_PI_SESSION_VERSION, + wrapEntry +} from '../src'; +import { assistantText, header, resetIds, userMessage } from './fixtures'; + +beforeEach(resetIds); + +describe('wrapEntry', () => { + it('wraps a pi entry in the four platform fields, leaving the entry untouched', () => { + const entry = userMessage('hello'); + const record = wrapEntry({ runId: 'run-1', seq: 1, entry, recordedAt: '2026-01-01T00:00:00.000Z' }); + + expect(record).toEqual({ + runId: 'run-1', + seq: 1, + recordedAt: '2026-01-01T00:00:00.000Z', + piSessionVersion: SUPPORTED_PI_SESSION_VERSION, + entry + }); + // Same object, not a copy: the entry is stored verbatim. + expect(record.entry).toBe(entry); + }); + + it('defaults recordedAt and records the pi session version', () => { + const record = wrapEntry({ runId: 'run-1', seq: 1, entry: userMessage('hi') }); + expect(Date.parse(record.recordedAt)).not.toBeNaN(); + expect(record.piSessionVersion).toBe(3); + expect(RUN_LOG_WRAPPER_VERSION).toBe(1); + }); + + it('rejects a seq that would break ordering', () => { + expect(() => wrapEntry({ runId: 'run-1', seq: 0, entry: userMessage('hi') })).toThrow(/positive integer/); + expect(() => wrapEntry({ runId: '', seq: 1, entry: userMessage('hi') })).toThrow(/runId/); + }); +}); + +describe('assertPiSessionEntry', () => { + it('accepts a session header without tree fields', () => { + expect(assertPiSessionEntry(header()).type).toBe('session'); + }); + + it('accepts an entry type it has never seen', () => { + const entry = { + type: 'quantum_thought', + id: 'abc', + parentId: null as string | null, + timestamp: '2026-01-01T00:00:00.000Z' + }; + expect(assertPiSessionEntry(entry)).toBe(entry); + }); + + it('throws rather than yielding an unreadable entry', () => { + expect(() => assertPiSessionEntry(null)).toThrow(/must be an object/); + expect(() => assertPiSessionEntry({})).toThrow(/`type`/); + expect(() => assertPiSessionEntry({ type: 'message' })).toThrow(/`id`/); + expect(() => assertPiSessionEntry({ type: 'message', id: 'a' })).toThrow(/timestamp/); + expect(() => assertPiSessionEntry({ type: 'message', id: 'a', timestamp: 'x' })).toThrow(/parentId/); + }); +}); + +describe('assertRunEventRecord', () => { + const valid = wrapEntry({ runId: 'run-1', seq: 1, entry: assistantText('hi') }); + + it('round-trips through JSON', () => { + expect(assertRunEventRecord(JSON.parse(JSON.stringify(valid)))).toEqual(valid); + }); + + it.each([ + [{ ...valid, runId: '' }, /non-empty runId/], + [{ ...valid, seq: 0 }, /invalid seq/], + [{ ...valid, recordedAt: 5 }, /recordedAt/], + [{ ...valid, piSessionVersion: '3' }, /piSessionVersion/], + [{ ...valid, entry: 'nope' }, /must be an object/] + ])('throws on a corrupt record (%#)', (record, message) => { + expect(() => assertRunEventRecord(record)).toThrow(message); + }); +}); + +describe('idempotencyKey', () => { + it('is stable per entry so a retried append is recognised', () => { + const entry = userMessage('hello'); + expect(idempotencyKey('run-1', entry)).toBe(idempotencyKey('run-1', { ...entry })); + expect(idempotencyKey('run-1', entry)).not.toBe(idempotencyKey('run-2', entry)); + }); +}); + +describe('assertOrdered', () => { + it('rejects mixed runs and out-of-order sequences', () => { + const a = wrapEntry({ runId: 'run-1', seq: 2, entry: userMessage('a') }); + const b = wrapEntry({ runId: 'run-1', seq: 1, entry: userMessage('b') }); + const c = wrapEntry({ runId: 'run-2', seq: 3, entry: userMessage('c') }); + expect(() => assertOrdered([a, b])).toThrow(/out of order/); + expect(() => assertOrdered([a, c])).toThrow(/mix runs/); + expect(() => assertOrdered([b, a])).not.toThrow(); + }); +}); diff --git a/agentic/run-log/__tests__/store.test.ts b/agentic/run-log/__tests__/store.test.ts new file mode 100644 index 0000000000..e288ebe37e --- /dev/null +++ b/agentic/run-log/__tests__/store.test.ts @@ -0,0 +1,135 @@ +import { cursorAfter, follow, MemoryRunLogStore, readAll, START } from '../src'; +import { assistantText, header, resetIds, userMessage } from './fixtures'; + +let store: MemoryRunLogStore; + +beforeEach(() => { + resetIds(); + store = new MemoryRunLogStore(); +}); + +describe('MemoryRunLogStore', () => { + it('assigns gapless sequences in append order', async () => { + await store.append('run-1', [header(), userMessage('hi')]); + await store.append('run-1', [assistantText('hello')]); + expect(store.snapshot('run-1').map((r) => r.seq)).toEqual([1, 2, 3]); + }); + + it('keeps runs independent', async () => { + await store.append('run-1', [userMessage('a')]); + await store.append('run-2', [userMessage('b')]); + expect(store.snapshot('run-1')).toHaveLength(1); + expect(store.snapshot('run-2')[0].seq).toBe(1); + expect(store.runIds()).toEqual(['run-1', 'run-2']); + }); + + it('skips entries it already holds, so an append can be retried', async () => { + const entries = [header(), userMessage('hi'), assistantText('hello')]; + const first = await store.append('run-1', entries); + const retry = await store.append('run-1', entries); + const extended = await store.append('run-1', [...entries, userMessage('again', 5)]); + + expect(first).toHaveLength(3); + expect(retry).toHaveLength(0); + expect(extended).toHaveLength(1); + expect(extended[0].seq).toBe(4); + }); + + it('reads after a cursor', async () => { + await store.append('run-1', [userMessage('a'), userMessage('b', 2), userMessage('c', 3)]); + + const first = await store.read('run-1', START, 2); + expect(first.records.map((r) => r.seq)).toEqual([1, 2]); + expect(first.cursor).toEqual({ afterSeq: 2 }); + + const next = await store.read('run-1', first.cursor); + expect(next.records.map((r) => r.seq)).toEqual([3]); + + const end = await store.read('run-1', next.cursor); + expect(end.records).toEqual([]); + // An empty page must not rewind the cursor. + expect(end.cursor).toEqual({ afterSeq: 3 }); + }); + + it('reports an empty page for an unknown run rather than throwing', async () => { + expect((await store.read('nope')).records).toEqual([]); + }); +}); + +describe('cursorAfter', () => { + it('keeps the previous position when nothing was read', () => { + expect(cursorAfter([], { afterSeq: 7 })).toEqual({ afterSeq: 7 }); + }); +}); + +describe('readAll', () => { + it('pages to the end of the run', async () => { + const entries = Array.from({ length: 25 }, (_, i) => userMessage(`m${String(i)}`, i + 1)); + await store.append('run-1', entries); + const records = await readAll(store, 'run-1', START, 10); + expect(records.map((r) => r.seq)).toEqual(entries.map((_, i) => i + 1)); + }); +}); + +describe('follow', () => { + const immediateSleep = async (): Promise => {}; + + it('yields batches as they arrive and stops on a terminal batch', async () => { + await store.append('run-1', [userMessage('a')]); + + const batches: number[][] = []; + const iteration = (async () => { + for await (const batch of follow(store, 'run-1', { + sleep: immediateSleep, + isTerminal: (records) => records.some((r) => r.entry.type === 'run_finished') + })) { + batches.push(batch.map((r) => r.seq)); + if (batches.length === 1) { + await store.append('run-1', [assistantText('b', 2)]); + } else if (batches.length === 2) { + await store.append('run-1', [ + { type: 'run_finished', id: 'ffffffff', parentId: null, timestamp: '2026-01-01T00:00:09.000Z' } + ]); + } + } + })(); + + await iteration; + expect(batches).toEqual([[1], [2], [3]]); + }); + + it('stops when the caller aborts', async () => { + const controller = new AbortController(); + const batches: number[][] = []; + const iteration = (async () => { + for await (const batch of follow(store, 'run-1', { sleep: immediateSleep, signal: controller.signal })) { + batches.push(batch.map((r) => r.seq)); + controller.abort(); + } + })(); + + await store.append('run-1', [userMessage('a')]); + await iteration; + expect(batches.length).toBeLessThanOrEqual(1); + }); + + it('races a push wakeup against the poll delay', async () => { + let wakeups = 0; + const iteration = (async () => { + for await (const batch of follow(store, 'run-1', { + sleep: immediateSleep, + waitForChange: async () => { + wakeups += 1; + if (wakeups === 1) await store.append('run-1', [userMessage('a')]); + }, + isTerminal: () => true + })) { + expect(batch).toHaveLength(1); + } + })(); + + await iteration; + expect(wakeups).toBeGreaterThan(0); + expect(store.snapshot('run-1')).toHaveLength(1); + }); +}); diff --git a/agentic/run-log/jest.config.js b/agentic/run-log/jest.config.js new file mode 100644 index 0000000000..8a26efd6d0 --- /dev/null +++ b/agentic/run-log/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/run-log/package.json b/agentic/run-log/package.json new file mode 100644 index 0000000000..1a14d0cbb2 --- /dev/null +++ b/agentic/run-log/package.json @@ -0,0 +1,38 @@ +{ + "name": "@agentic-kit/run-log", + "version": "0.1.0", + "author": "Constructive ", + "description": "The append-only agent run log \u2014 pi session entries stored verbatim under a run/seq wrapper, with projections for transcript, usage, tool state and resumable sessions", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "run-log", + "constructive" + ] +} diff --git a/agentic/run-log/src/file-store.ts b/agentic/run-log/src/file-store.ts new file mode 100644 index 0000000000..4a01bf8de8 --- /dev/null +++ b/agentic/run-log/src/file-store.ts @@ -0,0 +1,123 @@ +/** + * `@agentic-kit/run-log/file-store` — the node-only JSONL store. + * + * A separate entry point because the package's main entry is imported by + * browsers, Electron renderers and Next client components; a `node:fs` import + * there breaks those bundles. Same split, same reason, as `12factor-env/dotenv`. + * + * The file holds one wrapped record per line, so a run is recoverable with + * `tail -f` and a partially-written last line is detectable rather than silently + * dropped. This is the store a local run uses before (or without) a database, + * and the one tests use when they want the log to survive a process restart. + */ + +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; + +import type { PiSessionEntry } from './pi-entry'; +import { projectSession, type SessionProjectionOptions } from './projectors/session'; +import { + assertRunEventRecord, + idempotencyKey, + type RunEventRecord, + SUPPORTED_PI_SESSION_VERSION, + wrapEntry +} from './record'; +import { + type AppendOptions, + cursorAfter, + type RunLogCursor, + type RunLogPage, + type RunLogStore, + START +} from './store'; + +export interface FileRunLogStoreOptions { + /** Absolute path of the log file. Parent directories are created. */ + path: string; +} + +export class FileRunLogStore implements RunLogStore { + private readonly path: string; + + constructor(options: FileRunLogStoreOptions) { + this.path = options.path; + } + + private load(): RunEventRecord[] { + if (!existsSync(this.path)) return []; + const contents = readFileSync(this.path, 'utf8'); + const records: RunEventRecord[] = []; + const lines = contents.split('\n'); + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i].trim(); + if (line.length === 0) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch (error) { + throw new Error( + `run log ${this.path} line ${String(i + 1)} is not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + records.push(assertRunEventRecord(parsed)); + } + return records; + } + + async append( + runId: string, + entries: readonly PiSessionEntry[], + options: AppendOptions = {} + ): Promise { + const existing = this.load().filter((record) => record.runId === runId); + const seen = new Set(existing.map((record) => idempotencyKey(runId, record.entry))); + const written: RunEventRecord[] = []; + + for (const entry of entries) { + const key = idempotencyKey(runId, entry); + if (seen.has(key)) continue; + written.push( + wrapEntry({ + runId, + seq: existing.length + written.length + 1, + entry, + ...(options.recordedAt ? { recordedAt: options.recordedAt } : {}), + piSessionVersion: options.piSessionVersion ?? SUPPORTED_PI_SESSION_VERSION + }) + ); + seen.add(key); + } + + if (written.length === 0) return written; + + mkdirSync(dirname(this.path), { recursive: true }); + if (!existsSync(this.path)) writeFileSync(this.path, '', { mode: 0o600 }); + appendFileSync(this.path, written.map((record) => JSON.stringify(record)).join('\n') + '\n'); + return written; + } + + async read(runId: string, cursor: RunLogCursor = START, limit?: number): Promise { + const after = this.load().filter((record) => record.runId === runId && record.seq > cursor.afterSeq); + const records = typeof limit === 'number' ? after.slice(0, limit) : after; + return { records, cursor: cursorAfter(records, cursor) }; + } +} + +/** + * Write the run's pi session file so `SessionManager.open` can resume it. This + * is the cloud→local (and local→local restart) resume path: project, write, + * hand the path to pi. + */ +export function writeSessionFile( + path: string, + records: readonly RunEventRecord[], + options: SessionProjectionOptions = {} +): string { + const projection = projectSession(records, options); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, projection.jsonl, { mode: 0o600 }); + return path; +} diff --git a/agentic/run-log/src/follow.ts b/agentic/run-log/src/follow.ts new file mode 100644 index 0000000000..3dbce60829 --- /dev/null +++ b/agentic/run-log/src/follow.ts @@ -0,0 +1,97 @@ +/** + * Following a run log. + * + * Every surface — a desktop chat pane, a web execution view, a CLI tail, the + * resume path — is the same reader: hold a cursor, ask for what came after it. + * That is what makes local and cloud runs indistinguishable to a UI, so this is + * the only reader loop in the system. + * + * A push wakeup (LISTEN/NOTIFY, IPC, websocket) is an optimisation, not a + * requirement: `waitForChange` short-circuits the delay when it resolves, and + * without one the loop degrades to polling at `pollIntervalMs`. + */ + +import type { RunEventRecord } from './record'; +import { cursorAfter, type RunLogCursor, type RunLogReadStore, START } from './store'; + +export interface FollowOptions { + cursor?: RunLogCursor; + /** Polling delay when no wakeup arrives. */ + pollIntervalMs?: number; + /** Records per read. */ + limit?: number; + /** Resolves when the run may have new records; races the poll delay. */ + waitForChange?: (signal?: AbortSignal) => Promise; + /** Stop following once this returns true for the batch just yielded. */ + isTerminal?: (records: readonly RunEventRecord[]) => boolean; + signal?: AbortSignal; + /** Injectable for tests; defaults to `setTimeout`. */ + sleep?: (ms: number, signal?: AbortSignal) => Promise; +} + +const defaultSleep = (ms: number, signal?: AbortSignal): Promise => + new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + resolve(); + }, + { once: true } + ); + }); + +/** + * Yield every batch of new records until the run reaches a terminal state or + * the caller aborts. Batches are yielded as read, so a caller can project + * incrementally rather than re-projecting the whole run per frame. + */ +export async function* follow( + store: RunLogReadStore, + runId: string, + options: FollowOptions = {} +): AsyncGenerator { + const pollIntervalMs = options.pollIntervalMs ?? 500; + const sleep = options.sleep ?? defaultSleep; + let cursor = options.cursor ?? START; + + while (!options.signal?.aborted) { + const page = await store.read(runId, cursor, options.limit); + if (page.records.length > 0) { + cursor = cursorAfter(page.records, cursor); + yield page.records; + if (options.isTerminal?.(page.records)) return; + // Drain before waiting: a burst of tool output should not be paced by the + // poll interval. + continue; + } + + if (options.waitForChange) { + await Promise.race([ + options.waitForChange(options.signal), + sleep(pollIntervalMs, options.signal) + ]); + } else { + await sleep(pollIntervalMs, options.signal); + } + } +} + +/** Read a run to its current end in one pass. */ +export async function readAll( + store: RunLogReadStore, + runId: string, + cursor: RunLogCursor = START, + pageLimit = 500 +): Promise { + const records: RunEventRecord[] = []; + let position = cursor; + for (;;) { + const page = await store.read(runId, position, pageLimit); + if (page.records.length === 0) return records; + records.push(...page.records); + position = cursorAfter(page.records, position); + if (page.records.length < pageLimit) return records; + } +} diff --git a/agentic/run-log/src/index.ts b/agentic/run-log/src/index.ts new file mode 100644 index 0000000000..cb51aa22d1 --- /dev/null +++ b/agentic/run-log/src/index.ts @@ -0,0 +1,106 @@ +/** + * `@agentic-kit/run-log` — the append-only run log: one ordered record of what + * an agent run did, wherever it ran. + * + * Browser-safe on purpose: renderers import the projectors, so nothing here may + * reach for a node builtin. The filesystem store lives behind + * `@agentic-kit/run-log/file-store`. + */ + +export { + follow, + type FollowOptions, + readAll +} from './follow'; +export { + assertPiSessionEntry, + contentText, + isAssistantMessage, + isPiBranchSummaryEntry, + isPiCompactionEntry, + isPiMessageEntry, + isPiSessionHeader, + isToolResultMessage, + type PiAssistantMessage, + type PiBashExecutionMessage, + type PiBranchSummaryEntry, + type PiCompactionEntry, + type PiContent, + type PiCustomMessage, + type PiEntryBase, + type PiImageContent, + type PiMessage, + type PiMessageEntry, + type PiOtherEntry, + type PiSessionEntry, + type PiSessionHeader, + type PiSummaryMessage, + type PiTextContent, + type PiThinkingContent, + type PiToolCallContent, + type PiToolResultMessage, + type PiUsage, + type PiUsageCost, + type PiUserMessage, + toolCalls +} from './pi-entry'; +export { + type BashPart, + type Conversation, + type ConversationPart, + type CustomPart, + projectParts, + type SummaryPart, + type TextPart, + type ThinkingPart, + type ToolPart, + type ToolStatus, + type UnknownPart +} from './projectors/parts'; +export { + parseSessionJsonl, + projectSession, + type SessionProjection, + type SessionProjectionOptions +} from './projectors/session'; +export { + APPROVAL_REQUEST_TYPE, + APPROVAL_RESOLUTION_TYPE, + type ApprovalRequestInput, + approvalRequestMessage, + type ApprovalResolutionInput, + approvalResolutionMessage, + type ApprovalState, + projectToolState, + type ToolCallState, + type ToolCallStatus, + type ToolStateProjection +} from './projectors/tool-state'; +export { + modelKey, + type ModelUsage, + projectUsage, + type RunUsage, + type UsageTotals +} from './projectors/usage'; +export { + assertOrdered, + assertRunEventRecord, + idempotencyKey, + RUN_LOG_WRAPPER_VERSION, + type RunEventRecord, + SUPPORTED_PI_SESSION_VERSION, + wrapEntry, + type WrapEntryOptions +} from './record'; +export { + type AppendOptions, + cursorAfter, + MemoryRunLogStore, + type RunLogAppendStore, + type RunLogCursor, + type RunLogPage, + type RunLogReadStore, + type RunLogStore, + START +} from './store'; diff --git a/agentic/run-log/src/pi-entry.ts b/agentic/run-log/src/pi-entry.ts new file mode 100644 index 0000000000..8e3f900869 --- /dev/null +++ b/agentic/run-log/src/pi-entry.ts @@ -0,0 +1,238 @@ +/** + * The structural subset of pi's session entries this package reads. + * + * Deliberately structural, not imported from pi: a run log stores pi entries + * *verbatim*, so the types here describe what the projectors read rather than + * re-declaring pi's format. Every interface keeps an index signature so an + * entry produced by a newer pi still parses, and unknown `type` values are + * carried through untouched instead of being dropped. + * + * Reference: `@earendil-works/pi-coding-agent` `docs/session-format.md` + * (session file version 3). + */ + +export interface PiUsageCost { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + total?: number; +} + +export interface PiUsage { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + totalTokens?: number; + cost?: PiUsageCost; + [key: string]: unknown; +} + +export interface PiTextContent { + type: 'text'; + text: string; +} + +export interface PiThinkingContent { + type: 'thinking'; + thinking: string; +} + +export interface PiImageContent { + type: 'image'; + data: string; + mimeType: string; +} + +export interface PiToolCallContent { + type: 'toolCall'; + id: string; + name: string; + arguments?: Record; +} + +export type PiContent = PiTextContent | PiThinkingContent | PiImageContent | PiToolCallContent; + +export interface PiUserMessage { + role: 'user'; + content: string | PiContent[]; + timestamp?: number; + [key: string]: unknown; +} + +export interface PiAssistantMessage { + role: 'assistant'; + content: PiContent[]; + api?: string; + provider?: string; + model?: string; + usage?: PiUsage; + stopReason?: 'stop' | 'length' | 'toolUse' | 'error' | 'aborted'; + errorMessage?: string; + timestamp?: number; + [key: string]: unknown; +} + +export interface PiToolResultMessage { + role: 'toolResult'; + toolCallId: string; + toolName: string; + content: PiContent[]; + details?: unknown; + usage?: PiUsage; + isError?: boolean; + timestamp?: number; + [key: string]: unknown; +} + +export interface PiBashExecutionMessage { + role: 'bashExecution'; + command: string; + output: string; + exitCode?: number; + cancelled?: boolean; + truncated?: boolean; + timestamp?: number; + [key: string]: unknown; +} + +export interface PiCustomMessage { + role: 'custom'; + customType: string; + content: string | PiContent[]; + display?: boolean; + details?: unknown; + timestamp?: number; + [key: string]: unknown; +} + +export interface PiSummaryMessage { + role: 'branchSummary' | 'compactionSummary'; + summary: string; + timestamp?: number; + [key: string]: unknown; +} + +export type PiMessage = + | PiUserMessage + | PiAssistantMessage + | PiToolResultMessage + | PiBashExecutionMessage + | PiCustomMessage + | PiSummaryMessage; + +export interface PiSessionHeader { + type: 'session'; + version: number; + id: string; + timestamp: string; + cwd?: string; + parentSession?: string; + [key: string]: unknown; +} + +export interface PiEntryBase { + id: string; + parentId: string | null; + timestamp: string; +} + +export interface PiMessageEntry extends PiEntryBase { + type: 'message'; + message: PiMessage; + [key: string]: unknown; +} + +export interface PiCompactionEntry extends PiEntryBase { + type: 'compaction'; + summary: string; + tokensBefore?: number; + usage?: PiUsage; + [key: string]: unknown; +} + +export interface PiBranchSummaryEntry extends PiEntryBase { + type: 'branch_summary'; + summary: string; + fromId: string; + usage?: PiUsage; + [key: string]: unknown; +} + +export interface PiOtherEntry extends PiEntryBase { + type: string; + [key: string]: unknown; +} + +export type PiSessionEntry = + | PiSessionHeader + | PiMessageEntry + | PiCompactionEntry + | PiBranchSummaryEntry + | PiOtherEntry; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +export const isPiSessionHeader = (entry: PiSessionEntry): entry is PiSessionHeader => + entry.type === 'session'; + +export const isPiMessageEntry = (entry: PiSessionEntry): entry is PiMessageEntry => + entry.type === 'message' && isRecord((entry as PiMessageEntry).message); + +export const isPiCompactionEntry = (entry: PiSessionEntry): entry is PiCompactionEntry => + entry.type === 'compaction'; + +export const isPiBranchSummaryEntry = (entry: PiSessionEntry): entry is PiBranchSummaryEntry => + entry.type === 'branch_summary'; + +export const isAssistantMessage = (message: PiMessage): message is PiAssistantMessage => + message.role === 'assistant'; + +export const isToolResultMessage = (message: PiMessage): message is PiToolResultMessage => + message.role === 'toolResult'; + +/** + * Narrow an untrusted value (a database JSONB column, an HTTP body) to a pi + * entry. Throws rather than returning null: a log row that cannot be read is a + * corrupted log, never an empty one. + */ +export function assertPiSessionEntry(value: unknown): PiSessionEntry { + if (!isRecord(value)) { + throw new TypeError(`pi session entry must be an object, received ${typeof value}`); + } + if (typeof value.type !== 'string' || value.type.length === 0) { + throw new TypeError('pi session entry must carry a non-empty string `type`'); + } + if (value.type !== 'session') { + if (typeof value.id !== 'string' || value.id.length === 0) { + throw new TypeError(`pi ${value.type} entry must carry a non-empty string \`id\``); + } + if (typeof value.timestamp !== 'string') { + throw new TypeError(`pi ${value.type} entry must carry an ISO string \`timestamp\``); + } + if (!('parentId' in value)) { + throw new TypeError(`pi ${value.type} entry must carry \`parentId\` (null for the first entry)`); + } + } + return value as PiSessionEntry; +} + +/** The text of a message's content, whether it is a string or a block array. */ +export function contentText(content: string | PiContent[] | undefined): string { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter((block): block is PiTextContent => isRecord(block) && block.type === 'text') + .map((block) => block.text) + .join(''); +} + +/** Tool calls requested by an assistant message, in order. */ +export function toolCalls(message: PiAssistantMessage): PiToolCallContent[] { + if (!Array.isArray(message.content)) return []; + return message.content.filter( + (block): block is PiToolCallContent => isRecord(block) && block.type === 'toolCall' + ); +} diff --git a/agentic/run-log/src/projectors/parts.ts b/agentic/run-log/src/projectors/parts.ts new file mode 100644 index 0000000000..64593054f8 --- /dev/null +++ b/agentic/run-log/src/projectors/parts.ts @@ -0,0 +1,234 @@ +/** + * Renderable projection: run log records → an ordered list of parts a UI draws. + * + * This is the projection that replaces per-host transcript encodings. A tool + * call and its later result collapse into one part, so a renderer never has to + * correlate two messages itself, and an unknown entry type becomes an + * `unknown` part rather than disappearing — a log written by a newer pi still + * renders, minus the detail this version understands. + */ + +import { + contentText, + isAssistantMessage, + isPiBranchSummaryEntry, + isPiCompactionEntry, + isPiMessageEntry, + isPiSessionHeader, + isToolResultMessage, + type PiSessionEntry, + toolCalls +} from '../pi-entry'; +import type { RunEventRecord } from '../record'; + +export type ToolStatus = 'requested' | 'completed' | 'failed'; + +export interface PartBase { + /** Sequence of the record that introduced the part — a stable React key. */ + seq: number; + entryId?: string; +} + +export interface TextPart extends PartBase { + kind: 'text'; + role: 'user' | 'assistant'; + text: string; + model?: string; + provider?: string; +} + +export interface ThinkingPart extends PartBase { + kind: 'thinking'; + text: string; +} + +export interface ToolPart extends PartBase { + kind: 'tool'; + toolCallId: string; + name: string; + arguments: Record; + status: ToolStatus; + /** Text of the tool result, once one has been logged. */ + output?: string; + details?: unknown; + /** Sequence of the record that settled the call. */ + settledSeq?: number; +} + +export interface BashPart extends PartBase { + kind: 'bash'; + command: string; + output: string; + exitCode?: number; +} + +export interface CustomPart extends PartBase { + kind: 'custom'; + customType: string; + text: string; + display: boolean; + details?: unknown; +} + +export interface SummaryPart extends PartBase { + kind: 'summary'; + reason: 'compaction' | 'branch'; + summary: string; +} + +export interface UnknownPart extends PartBase { + kind: 'unknown'; + entryType: string; + entry: PiSessionEntry; +} + +export type ConversationPart = + | TextPart + | ThinkingPart + | ToolPart + | BashPart + | CustomPart + | SummaryPart + | UnknownPart; + +export interface Conversation { + parts: ConversationPart[]; + /** From the session header, when the log carries one. */ + sessionId?: string; + cwd?: string; +} + +/** + * Project records into a conversation. Pure and total: the same records always + * produce the same parts, whichever host wrote them. + */ +export function projectParts(records: readonly RunEventRecord[]): Conversation { + const parts: ConversationPart[] = []; + const toolsByCallId = new Map(); + let sessionId: string | undefined; + let cwd: string | undefined; + + for (const record of records) { + const { entry, seq } = record; + + if (isPiSessionHeader(entry)) { + sessionId = entry.id; + if (typeof entry.cwd === 'string') cwd = entry.cwd; + continue; + } + + if (isPiCompactionEntry(entry)) { + parts.push({ kind: 'summary', reason: 'compaction', summary: entry.summary, seq, entryId: entry.id }); + continue; + } + + if (isPiBranchSummaryEntry(entry)) { + parts.push({ kind: 'summary', reason: 'branch', summary: entry.summary, seq, entryId: entry.id }); + continue; + } + + if (!isPiMessageEntry(entry)) { + parts.push({ kind: 'unknown', entryType: entry.type, entry, seq, entryId: (entry as { id?: string }).id }); + continue; + } + + const message = entry.message; + const base = { seq, entryId: entry.id }; + + if (message.role === 'user') { + parts.push({ kind: 'text', role: 'user', text: contentText(message.content), ...base }); + continue; + } + + if (isAssistantMessage(message)) { + for (const block of Array.isArray(message.content) ? message.content : []) { + if (block.type === 'text' && block.text.length > 0) { + parts.push({ + kind: 'text', + role: 'assistant', + text: block.text, + ...(message.model ? { model: message.model } : {}), + ...(message.provider ? { provider: message.provider } : {}), + ...base + }); + } else if (block.type === 'thinking') { + parts.push({ kind: 'thinking', text: block.thinking, ...base }); + } + } + for (const call of toolCalls(message)) { + const part: ToolPart = { + kind: 'tool', + toolCallId: call.id, + name: call.name, + arguments: call.arguments ?? {}, + status: 'requested', + ...base + }; + toolsByCallId.set(call.id, part); + parts.push(part); + } + continue; + } + + if (isToolResultMessage(message)) { + const existing = toolsByCallId.get(message.toolCallId); + const output = contentText(message.content); + const status: ToolStatus = message.isError ? 'failed' : 'completed'; + if (existing) { + existing.status = status; + existing.output = output; + existing.settledSeq = seq; + if (message.details !== undefined) existing.details = message.details; + } else { + // A result whose call is not in this window (paged read, forked branch). + parts.push({ + kind: 'tool', + toolCallId: message.toolCallId, + name: message.toolName, + arguments: {}, + status, + output, + settledSeq: seq, + ...base + }); + } + continue; + } + + if (message.role === 'bashExecution') { + parts.push({ + kind: 'bash', + command: message.command, + output: message.output, + ...(typeof message.exitCode === 'number' ? { exitCode: message.exitCode } : {}), + ...base + }); + continue; + } + + if (message.role === 'custom') { + parts.push({ + kind: 'custom', + customType: message.customType, + text: contentText(message.content), + display: message.display !== false, + ...(message.details !== undefined ? { details: message.details } : {}), + ...base + }); + continue; + } + + parts.push({ + kind: 'summary', + reason: message.role === 'branchSummary' ? 'branch' : 'compaction', + summary: (message as { summary?: string }).summary ?? '', + ...base + }); + } + + return { + parts, + ...(sessionId ? { sessionId } : {}), + ...(cwd ? { cwd } : {}) + }; +} diff --git a/agentic/run-log/src/projectors/session.ts b/agentic/run-log/src/projectors/session.ts new file mode 100644 index 0000000000..abd53677a8 --- /dev/null +++ b/agentic/run-log/src/projectors/session.ts @@ -0,0 +1,94 @@ +/** + * Session projection: run log records → a pi session file. + * + * This is what makes a run resumable anywhere. The log is the source of truth; + * a `.jsonl` session is a derived artifact, so a run that started in the cloud + * can be continued locally (and vice versa) by projecting the log back into the + * only format pi's `SessionManager` reads. + * + * Returns a string rather than writing a file: the projection is pure, and the + * node-side write lives in `@agentic-kit/run-log/file-store`. + */ + +import { isPiSessionHeader, type PiSessionEntry } from '../pi-entry'; +import { assertOrdered, type RunEventRecord, SUPPORTED_PI_SESSION_VERSION } from '../record'; + +export interface SessionProjectionOptions { + /** Used when the log carries no session header (a run logged headerless). */ + sessionId?: string; + cwd?: string; + timestamp?: string; +} + +export interface SessionProjection { + /** The full session file contents, newline-terminated. */ + jsonl: string; + /** Entries in file order, header first. */ + entries: PiSessionEntry[]; + piSessionVersion: number; +} + +/** + * Project records into a pi session file. Throws when the records cannot form a + * loadable session — an unresumable session must fail at projection time, not + * when pi later reads a truncated tree. + */ +export function projectSession( + records: readonly RunEventRecord[], + options: SessionProjectionOptions = {} +): SessionProjection { + assertOrdered(records); + + const versions = new Set(records.map((record) => record.piSessionVersion)); + if (versions.size > 1) { + throw new Error( + `run log mixes pi session versions (${Array.from(versions).sort().join(', ')}); migrate the older entries before projecting a session` + ); + } + const piSessionVersion = records[0]?.piSessionVersion ?? SUPPORTED_PI_SESSION_VERSION; + + const entries = records.map((record) => record.entry); + const headerIndex = entries.findIndex(isPiSessionHeader); + if (headerIndex > 0) { + throw new Error( + `run log carries a session header at position ${String(headerIndex)}; a pi session file requires it first` + ); + } + + const body = headerIndex === 0 ? entries.slice(1) : entries; + const header: PiSessionEntry = + headerIndex === 0 + ? entries[0] + : { + type: 'session', + version: piSessionVersion, + id: options.sessionId ?? records[0]?.runId ?? 'run-log', + timestamp: options.timestamp ?? records[0]?.recordedAt ?? new Date().toISOString(), + ...(options.cwd ? { cwd: options.cwd } : {}) + }; + + const ordered = [header, ...body]; + return { + entries: ordered, + piSessionVersion, + jsonl: ordered.map((entry) => JSON.stringify(entry)).join('\n') + '\n' + }; +} + +/** Parse a pi session file into entries — the inverse, for importing a session. */ +export function parseSessionJsonl(jsonl: string): PiSessionEntry[] { + const entries: PiSessionEntry[] = []; + const lines = jsonl.split('\n'); + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i].trim(); + if (line.length === 0) continue; + try { + entries.push(JSON.parse(line) as PiSessionEntry); + } catch (error) { + throw new Error( + `pi session line ${String(i + 1)} is not valid JSON: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + return entries; +} diff --git a/agentic/run-log/src/projectors/tool-state.ts b/agentic/run-log/src/projectors/tool-state.ts new file mode 100644 index 0000000000..35204543b2 --- /dev/null +++ b/agentic/run-log/src/projectors/tool-state.ts @@ -0,0 +1,192 @@ +/** + * Tool and approval projection: run log records → the state a UI needs to know + * what is running and what is waiting on a human. + * + * Approvals ride in the log as pi `custom` messages rather than a side channel, + * because "the run is blocked on you" is part of the run's history: a surface + * that reconnects hours later must be able to see a pending request without + * having been present when it was raised, and both placements then behave the + * same — the cloud already treats the conversation as the approval UI. + */ + +import { contentText, isAssistantMessage, isPiMessageEntry, isToolResultMessage, toolCalls } from '../pi-entry'; +import type { RunEventRecord } from '../record'; + +/** `customType` of an approval request written by the gate extension. */ +export const APPROVAL_REQUEST_TYPE = 'constructive.approval.request'; +/** `customType` of the human's answer to a request. */ +export const APPROVAL_RESOLUTION_TYPE = 'constructive.approval.resolution'; + +export type ToolCallStatus = 'requested' | 'awaiting-approval' | 'rejected' | 'running' | 'completed' | 'failed'; + +export interface ToolCallState { + toolCallId: string; + name: string; + arguments: Record; + status: ToolCallStatus; + requestedSeq: number; + settledSeq?: number; + output?: string; + approval?: ApprovalState; +} + +export interface ApprovalState { + toolCallId: string; + requestedSeq: number; + prompt: string; + resolvedSeq?: number; + decision?: 'approved' | 'rejected'; + reason?: string; + actorId?: string; +} + +export interface ToolStateProjection { + tools: Record; + /** In log order — the oldest unanswered request first. */ + pendingApprovals: ApprovalState[]; +} + +interface ApprovalDetails { + toolCallId?: unknown; + decision?: unknown; + reason?: unknown; + actorId?: unknown; +} + +const details = (value: unknown): ApprovalDetails => + typeof value === 'object' && value !== null ? (value as ApprovalDetails) : {}; + +export function projectToolState(records: readonly RunEventRecord[]): ToolStateProjection { + const tools: Record = {}; + const approvals = new Map(); + + for (const { entry, seq } of records) { + if (!isPiMessageEntry(entry)) continue; + const message = entry.message; + + if (isAssistantMessage(message)) { + for (const call of toolCalls(message)) { + tools[call.id] = { + toolCallId: call.id, + name: call.name, + arguments: call.arguments ?? {}, + status: 'requested', + requestedSeq: seq + }; + } + continue; + } + + if (isToolResultMessage(message)) { + const state = tools[message.toolCallId]; + const settled: Partial = { + status: message.isError ? 'failed' : 'completed', + settledSeq: seq, + output: contentText(message.content) + }; + tools[message.toolCallId] = state + ? { ...state, ...settled } + : { + toolCallId: message.toolCallId, + name: message.toolName, + arguments: {}, + requestedSeq: seq, + status: settled.status as ToolCallStatus, + settledSeq: seq, + output: settled.output as string + }; + continue; + } + + if (message.role !== 'custom') continue; + + if (message.customType === APPROVAL_REQUEST_TYPE) { + const info = details(message.details); + const toolCallId = typeof info.toolCallId === 'string' ? info.toolCallId : null; + if (!toolCallId) { + throw new Error( + `approval request at seq ${String(seq)} carries no toolCallId; the run log cannot attribute it` + ); + } + const approval: ApprovalState = { + toolCallId, + requestedSeq: seq, + prompt: contentText(message.content) + }; + approvals.set(toolCallId, approval); + const state = tools[toolCallId]; + if (state) tools[toolCallId] = { ...state, status: 'awaiting-approval', approval }; + continue; + } + + if (message.customType === APPROVAL_RESOLUTION_TYPE) { + const info = details(message.details); + const toolCallId = typeof info.toolCallId === 'string' ? info.toolCallId : null; + if (!toolCallId) { + throw new Error( + `approval resolution at seq ${String(seq)} carries no toolCallId; the run log cannot attribute it` + ); + } + const approved = info.decision === 'approved'; + const existing = approvals.get(toolCallId); + const approval: ApprovalState = { + ...(existing ?? { toolCallId, requestedSeq: seq, prompt: '' }), + resolvedSeq: seq, + decision: approved ? 'approved' : 'rejected', + ...(typeof info.reason === 'string' ? { reason: info.reason } : {}), + ...(typeof info.actorId === 'string' ? { actorId: info.actorId } : {}) + }; + approvals.set(toolCallId, approval); + const state = tools[toolCallId]; + if (state && state.status === 'awaiting-approval') { + tools[toolCallId] = { ...state, status: approved ? 'running' : 'rejected', approval }; + } else if (state) { + tools[toolCallId] = { ...state, approval }; + } + } + } + + const pendingApprovals = Array.from(approvals.values()) + .filter((approval) => approval.resolvedSeq === undefined) + .sort((a, b) => a.requestedSeq - b.requestedSeq); + + return { tools, pendingApprovals }; +} + +/** The parts of an approval request an extension needs to write one. */ +export interface ApprovalRequestInput { + toolCallId: string; + prompt: string; +} + +export interface ApprovalResolutionInput { + toolCallId: string; + decision: 'approved' | 'rejected'; + reason?: string; + actorId?: string; +} + +/** Build the pi `custom` message an approval request is carried in. */ +export const approvalRequestMessage = (input: ApprovalRequestInput) => ({ + role: 'custom' as const, + customType: APPROVAL_REQUEST_TYPE, + content: input.prompt, + display: true, + details: { toolCallId: input.toolCallId }, + timestamp: Date.now() +}); + +/** Build the pi `custom` message a human's answer is carried in. */ +export const approvalResolutionMessage = (input: ApprovalResolutionInput) => ({ + role: 'custom' as const, + customType: APPROVAL_RESOLUTION_TYPE, + content: input.reason ?? input.decision, + display: true, + details: { + toolCallId: input.toolCallId, + decision: input.decision, + ...(input.reason ? { reason: input.reason } : {}), + ...(input.actorId ? { actorId: input.actorId } : {}) + }, + timestamp: Date.now() +}); diff --git a/agentic/run-log/src/projectors/usage.ts b/agentic/run-log/src/projectors/usage.ts new file mode 100644 index 0000000000..9e150d3ad8 --- /dev/null +++ b/agentic/run-log/src/projectors/usage.ts @@ -0,0 +1,113 @@ +/** + * Usage projection: run log records → token and cost totals. + * + * Every usage-bearing pi entry counts, not just assistant messages: a tool that + * performed nested LLM work reports `usage` on its result, and compaction and + * branch summaries are model calls the run paid for. Missing any of those makes + * a run look cheaper than it was, which is exactly the kind of drift metering + * exists to prevent. + * + * This projection describes what a run *observed*. It is reconciliation input, + * never the billing authority — a gateway-observed record is (see + * `@agentic-kit/pi-ext-metered-model`). + */ + +import { + isAssistantMessage, + isPiBranchSummaryEntry, + isPiCompactionEntry, + isPiMessageEntry, + isToolResultMessage, + type PiUsage +} from '../pi-entry'; +import type { RunEventRecord } from '../record'; + +export interface UsageTotals { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + cost: number; + /** Number of usage-bearing entries folded into these totals. */ + calls: number; +} + +export interface ModelUsage extends UsageTotals { + provider: string; + model: string; +} + +export interface RunUsage extends UsageTotals { + /** Per provider+model breakdown, keyed `provider/model`. */ + byModel: Record; +} + +const empty = (): UsageTotals => ({ + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: 0, + calls: 0 +}); + +const num = (value: unknown): number => (typeof value === 'number' && Number.isFinite(value) ? value : 0); + +function fold(target: UsageTotals, usage: PiUsage): void { + const input = num(usage.input); + const output = num(usage.output); + const cacheRead = num(usage.cacheRead); + const cacheWrite = num(usage.cacheWrite); + target.input += input; + target.output += output; + target.cacheRead += cacheRead; + target.cacheWrite += cacheWrite; + // Providers that omit a total still have one: the parts they did report. + target.totalTokens += usage.totalTokens === undefined + ? input + output + cacheRead + cacheWrite + : num(usage.totalTokens); + target.cost += num(usage.cost?.total); + target.calls += 1; +} + +export const modelKey = (provider: string, model: string): string => `${provider}/${model}`; + +/** Fold every usage-bearing entry in the records into run totals. */ +export function projectUsage(records: readonly RunEventRecord[]): RunUsage { + const totals: RunUsage = { ...empty(), byModel: {} }; + + const add = (usage: PiUsage | undefined, provider = 'unknown', model = 'unknown'): void => { + if (!usage) return; + fold(totals, usage); + const key = modelKey(provider, model); + const bucket = totals.byModel[key] ?? { ...empty(), provider, model }; + fold(bucket, usage); + totals.byModel[key] = bucket; + }; + + let lastProvider = 'unknown'; + let lastModel = 'unknown'; + + for (const { entry } of records) { + if (isPiMessageEntry(entry)) { + const message = entry.message; + if (isAssistantMessage(message)) { + lastProvider = message.provider ?? lastProvider; + lastModel = message.model ?? lastModel; + add(message.usage, message.provider ?? 'unknown', message.model ?? 'unknown'); + } else if (isToolResultMessage(message)) { + // Nested model work inside a tool: attributed to the run's current model, + // which is the model that requested the tool. + add(message.usage, lastProvider, lastModel); + } + continue; + } + if (isPiCompactionEntry(entry) || isPiBranchSummaryEntry(entry)) { + add(entry.usage, lastProvider, lastModel); + } + } + + return totals; +} diff --git a/agentic/run-log/src/record.ts b/agentic/run-log/src/record.ts new file mode 100644 index 0000000000..938437285a --- /dev/null +++ b/agentic/run-log/src/record.ts @@ -0,0 +1,115 @@ +/** + * The run log record: four platform-owned fields around a verbatim pi entry. + * + * The wrapper exists to give an entry identity (which run) and order (which + * position) across hosts — nothing else. It deliberately does not re-encode pi's + * semantics, because pi already versions and migrates its own session format; + * `piSessionVersion` records which version an entry was written under so a + * reader can hand old entries to pi's migrations instead of guessing. + */ + +import { assertPiSessionEntry, type PiSessionEntry } from './pi-entry'; + +/** Version of the wrapper itself — bumped only if these four fields change. */ +export const RUN_LOG_WRAPPER_VERSION = 1; + +/** The pi session format version this package projects without migration. */ +export const SUPPORTED_PI_SESSION_VERSION = 3; + +export interface RunEventRecord { + /** The run this entry belongs to. */ + runId: string; + /** 1-based position within the run. Gapless and strictly increasing. */ + seq: number; + /** When the platform durably recorded the entry (ISO 8601). */ + recordedAt: string; + /** pi's session format version at write time. */ + piSessionVersion: number; + /** The pi session entry, byte-for-byte as pi produced it. */ + entry: PiSessionEntry; +} + +export interface WrapEntryOptions { + runId: string; + seq: number; + entry: PiSessionEntry; + recordedAt?: string; + piSessionVersion?: number; +} + +export function wrapEntry(options: WrapEntryOptions): RunEventRecord { + if (!options.runId) throw new TypeError('a run log record needs a runId'); + if (!Number.isInteger(options.seq) || options.seq < 1) { + throw new TypeError(`run log seq must be a positive integer, received ${String(options.seq)}`); + } + return { + runId: options.runId, + seq: options.seq, + recordedAt: options.recordedAt ?? new Date().toISOString(), + piSessionVersion: options.piSessionVersion ?? SUPPORTED_PI_SESSION_VERSION, + entry: assertPiSessionEntry(options.entry) + }; +} + +/** + * Narrow an untrusted record (database row, HTTP body). Throws on anything + * unreadable — a log that cannot be parsed must fail loudly, never silently + * render as an empty conversation. + */ +export function assertRunEventRecord(value: unknown): RunEventRecord { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new TypeError(`run log record must be an object, received ${typeof value}`); + } + const record = value as Record; + if (typeof record.runId !== 'string' || record.runId.length === 0) { + throw new TypeError('run log record must carry a non-empty runId'); + } + if (!Number.isInteger(record.seq) || (record.seq as number) < 1) { + throw new TypeError(`run log record ${record.runId} has an invalid seq: ${String(record.seq)}`); + } + if (typeof record.recordedAt !== 'string') { + throw new TypeError(`run log record ${record.runId}#${String(record.seq)} must carry recordedAt`); + } + if (!Number.isInteger(record.piSessionVersion)) { + throw new TypeError( + `run log record ${record.runId}#${String(record.seq)} must carry an integer piSessionVersion` + ); + } + return { + runId: record.runId, + seq: record.seq as number, + recordedAt: record.recordedAt, + piSessionVersion: record.piSessionVersion as number, + entry: assertPiSessionEntry(record.entry) + }; +} + +/** + * The de-duplication key for an append. pi entry ids are unique within a + * session, so a retried append (a Job restart re-emitting its tail, a + * reconnecting writer) is recognised rather than duplicated. + */ +export function idempotencyKey(runId: string, entry: PiSessionEntry): string { + const id = typeof (entry as { id?: unknown }).id === 'string' ? (entry as { id: string }).id : null; + if (id) return `${runId}:${entry.type}:${id}`; + // Session headers carry no tree id; there is exactly one per session file. + return `${runId}:${entry.type}:${String((entry as { id?: string }).id ?? 'header')}`; +} + +/** Records must be contiguous and in order before anything projects them. */ +export function assertOrdered(records: readonly RunEventRecord[]): void { + for (let i = 1; i < records.length; i += 1) { + const previous = records[i - 1]; + const current = records[i]; + if (current.runId !== previous.runId) { + throw new Error( + `run log records mix runs: ${previous.runId} then ${current.runId} at index ${String(i)}` + ); + } + if (current.seq <= previous.seq) { + throw new Error( + `run log records out of order in ${current.runId}: seq ${String(previous.seq)} followed by ${String(current.seq)}` + ); + } + } +} diff --git a/agentic/run-log/src/store.ts b/agentic/run-log/src/store.ts new file mode 100644 index 0000000000..d95d72f760 --- /dev/null +++ b/agentic/run-log/src/store.ts @@ -0,0 +1,103 @@ +/** + * The storage contract. A run log is append-only and read by cursor, so the + * interfaces are deliberately two: a writer (the agent host) and a reader + * (every UI surface, the resume path, the usage rollup). Concrete stores live + * with their storage — Postgres in constructive-db, JSONL in `./file-store`, + * memory here for tests and for a run that has not been persisted yet. + */ + +import type { PiSessionEntry } from './pi-entry'; +import { + assertOrdered, + idempotencyKey, + type RunEventRecord, + SUPPORTED_PI_SESSION_VERSION, + wrapEntry +} from './record'; + +/** Read position: `afterSeq` is exclusive, so `0` means "from the start". */ +export interface RunLogCursor { + afterSeq: number; +} + +export const START: RunLogCursor = { afterSeq: 0 }; + +export const cursorAfter = (records: readonly RunEventRecord[], from: RunLogCursor = START): RunLogCursor => + records.length === 0 ? from : { afterSeq: records[records.length - 1].seq }; + +export interface RunLogPage { + records: RunEventRecord[]; + cursor: RunLogCursor; +} + +export interface AppendOptions { + /** pi session format version the entries were produced under. */ + piSessionVersion?: number; + recordedAt?: string; +} + +export interface RunLogAppendStore { + /** + * Append entries to a run, returning the records actually written. Entries + * already present (matched by pi entry id) are skipped, which makes an append + * safe to retry. + */ + append(runId: string, entries: readonly PiSessionEntry[], options?: AppendOptions): Promise; +} + +export interface RunLogReadStore { + read(runId: string, cursor?: RunLogCursor, limit?: number): Promise; +} + +export type RunLogStore = RunLogAppendStore & RunLogReadStore; + +/** In-memory store: the reference implementation and the test double. */ +export class MemoryRunLogStore implements RunLogStore { + private readonly runs = new Map(); + private readonly seen = new Map>(); + + async append( + runId: string, + entries: readonly PiSessionEntry[], + options: AppendOptions = {} + ): Promise { + const records = this.runs.get(runId) ?? []; + const seen = this.seen.get(runId) ?? new Set(); + const written: RunEventRecord[] = []; + + for (const entry of entries) { + const key = idempotencyKey(runId, entry); + if (seen.has(key)) continue; + const record = wrapEntry({ + runId, + seq: records.length + written.length + 1, + entry, + ...(options.recordedAt ? { recordedAt: options.recordedAt } : {}), + piSessionVersion: options.piSessionVersion ?? SUPPORTED_PI_SESSION_VERSION + }); + written.push(record); + seen.add(key); + } + + this.runs.set(runId, records.concat(written)); + this.seen.set(runId, seen); + return written; + } + + async read(runId: string, cursor: RunLogCursor = START, limit?: number): Promise { + const all = this.runs.get(runId) ?? []; + const after = all.filter((record) => record.seq > cursor.afterSeq); + const records = typeof limit === 'number' ? after.slice(0, limit) : after; + assertOrdered(records); + return { records, cursor: cursorAfter(records, cursor) }; + } + + /** Test/debug helper: every record of a run, ignoring cursors. */ + snapshot(runId: string): RunEventRecord[] { + return (this.runs.get(runId) ?? []).slice(); + } + + runIds(): string[] { + return Array.from(this.runs.keys()); + } +} diff --git a/agentic/run-log/tsconfig.esm.json b/agentic/run-log/tsconfig.esm.json new file mode 100644 index 0000000000..624ab17cfa --- /dev/null +++ b/agentic/run-log/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/run-log/tsconfig.json b/agentic/run-log/tsconfig.json new file mode 100644 index 0000000000..df063b5ee3 --- /dev/null +++ b/agentic/run-log/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7bdb4c757c..c266a1191e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -287,6 +287,9 @@ importers: version: 19.2.5(react@19.2.5) publishDirectory: dist + agentic/run-log: + publishDirectory: dist + examples/codegen-integration: dependencies: '@0no-co/graphql.web': From 68492fac801411f0cc434629b7a79a078f255c6f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 14 Aug 2026 03:56:17 +0000 Subject: [PATCH 2/5] feat(pi-ext-run-log): mirror pi session entries into the run log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pi extension that drains pi's append-only session into an @agentic-kit/run-log store — verbatim, in order, same code locally and in the cloud (only runId and the store differ). Draining is index-based because the session is append-only; the read position is keyed to the session header id so a switch/fork re-mirrors and the store's idempotency absorbs the duplicates. A drain advances only after a successful append, and store failures are rethrown into pi's dispatch by default rather than silently dropping entries. --- .github/workflows/run-tests.yaml | 2 +- agentic/pi-ext-run-log/README.md | 48 +++++ .../__tests__/extension.test.ts | 114 +++++++++++ .../pi-ext-run-log/__tests__/mirror.test.ts | 186 ++++++++++++++++++ agentic/pi-ext-run-log/jest.config.js | 21 ++ agentic/pi-ext-run-log/package.json | 47 +++++ agentic/pi-ext-run-log/src/extension.ts | 89 +++++++++ agentic/pi-ext-run-log/src/index.ts | 14 ++ agentic/pi-ext-run-log/src/mirror.ts | 107 ++++++++++ agentic/pi-ext-run-log/tsconfig.esm.json | 7 + agentic/pi-ext-run-log/tsconfig.json | 8 + pnpm-lock.yaml | 11 ++ 12 files changed, 653 insertions(+), 1 deletion(-) create mode 100644 agentic/pi-ext-run-log/README.md create mode 100644 agentic/pi-ext-run-log/__tests__/extension.test.ts create mode 100644 agentic/pi-ext-run-log/__tests__/mirror.test.ts create mode 100644 agentic/pi-ext-run-log/jest.config.js create mode 100644 agentic/pi-ext-run-log/package.json create mode 100644 agentic/pi-ext-run-log/src/extension.ts create mode 100644 agentic/pi-ext-run-log/src/index.ts create mode 100644 agentic/pi-ext-run-log/src/mirror.ts create mode 100644 agentic/pi-ext-run-log/tsconfig.esm.json create mode 100644 agentic/pi-ext-run-log/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index b76174b9b0..30ee5aaef4 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/pi-ext-run-log/README.md b/agentic/pi-ext-run-log/README.md new file mode 100644 index 0000000000..295ed45496 --- /dev/null +++ b/agentic/pi-ext-run-log/README.md @@ -0,0 +1,48 @@ +

+ +

+ +# @agentic-kit/pi-ext-run-log + +The write side of the [`@agentic-kit/run-log`](../run-log) for a pi coding-agent session: a pi extension that mirrors every session entry into a run-log store, verbatim and in order. + +The same extension runs in both placements — a local session in Constructive Desktop and a cloud session in a long-running job. Only `runId` and the store differ, which is what makes a run observable from anywhere without a second transcript format. + +## Usage + +```ts +import { createRunLogExtension } from '@agentic-kit/pi-ext-run-log'; + +const { extension, flush } = createRunLogExtension({ + runId, + store // any RunLogAppendStore: memory, JSONL, or the API/Postgres-backed one +}); + +const session = await AgentSession.create({ extensions: [extension] }); +// … +await flush(); // before the host exits +``` + +## How it mirrors + +pi owns the session — an append-only tree of entries, persisted as JSONL — and exposes no "entry appended" event. So the extension *drains*: after each event that could have appended, it takes the entries it has not seen yet and appends them. + +- Index-based draining is sound because the session is append-only: entries are never rewritten or removed, only branched from. +- The session header is mirrored once, as the first record of that session. +- Drains are serialized, so concurrent events cannot interleave batches and break run-log ordering. +- A drain advances its read position only after a successful append, so a failed append is retried whole rather than leaving a hole. +- Read position is keyed to the session header id: a switch/fork/new-session re-mirrors from the start, and entries carried over are absorbed by the store's idempotency (pi entry ids). The same property makes resume free — a restarted host re-appends its history and writes nothing new. + +`MIRROR_EVENTS` is the default event list; narrow it with `events` if a host wants fewer drains. + +## Failure behavior + +A store failure is **rethrown into pi's event dispatch** by default. A run log that silently stops recording is worse than a loud one, so losing entries is never the default. Pass `onError` if the host would rather log and continue. + +## Testing + +`SessionMirror` is the whole mechanism and knows nothing about pi's extension API, so mirroring is tested against a fake session; the extension test drives the registered handlers with a fake `ExtensionAPI`. + +```sh +pnpm test +``` diff --git a/agentic/pi-ext-run-log/__tests__/extension.test.ts b/agentic/pi-ext-run-log/__tests__/extension.test.ts new file mode 100644 index 0000000000..9c7797b187 --- /dev/null +++ b/agentic/pi-ext-run-log/__tests__/extension.test.ts @@ -0,0 +1,114 @@ +import { MemoryRunLogStore, type RunLogAppendStore, START } from '@agentic-kit/run-log'; +import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; + +import { createRunLogExtension, MIRROR_EVENTS } from '../src/extension'; + +const header = { type: 'session', version: 3, id: 'sess-1', timestamp: '2026-01-01T00:00:00.000Z', cwd: '/w' }; + +const entry = (id: string, parentId: string | null) => ({ + type: 'message', + id, + parentId, + timestamp: '2026-01-01T00:00:01.000Z', + message: { role: 'assistant', content: [{ type: 'text', text: id }] } +}); + +interface FakePi { + api: ExtensionAPI; + emit(event: string): Promise; + registered: string[]; + entries: unknown[]; +} + +const fakePi = (): FakePi => { + const handlers = new Map Promise | void>(); + const entries: unknown[] = []; + const sessionManager = { getHeader: () => header, getEntries: () => entries }; + const api = { + on: (event: string, handler: (event: unknown, ctx: unknown) => Promise | void) => { + handlers.set(event, handler); + } + } as unknown as ExtensionAPI; + + return { + api, + registered: [], + entries, + emit: async (event) => { + const handler = handlers.get(event); + if (!handler) throw new Error(`no handler registered for ${event}`); + await handler({ type: event }, { sessionManager }); + } + }; +}; + +describe('createRunLogExtension', () => { + it('drains the session on each mirrored event', async () => { + const store = new MemoryRunLogStore(); + const pi = fakePi(); + const { extension } = createRunLogExtension({ runId: 'run-1', store }); + extension(pi.api); + + pi.entries.push(entry('e1', null)); + await pi.emit('session_start'); + pi.entries.push(entry('e2', 'e1')); + await pi.emit('message_end'); + + const page = await store.read('run-1', START); + expect(page.records.map((r) => r.seq)).toEqual([1, 2, 3]); + }); + + it('registers every mirrored event, and only those', async () => { + const seen: string[] = []; + const api = { on: (event: string) => seen.push(event) } as unknown as ExtensionAPI; + createRunLogExtension({ runId: 'run-1', store: new MemoryRunLogStore() }).extension(api); + expect(seen).toEqual([...MIRROR_EVENTS]); + }); + + it('honours a narrowed event list', () => { + const seen: string[] = []; + const api = { on: (event: string) => seen.push(event) } as unknown as ExtensionAPI; + createRunLogExtension({ runId: 'run-1', store: new MemoryRunLogStore(), events: ['turn_end'] }).extension(api); + expect(seen).toEqual(['turn_end']); + }); + + it('throws into pi by default when the store fails', async () => { + const store: RunLogAppendStore = { + append: async () => { + throw new Error('store offline'); + } + }; + const pi = fakePi(); + createRunLogExtension({ runId: 'run-1', store }).extension(pi.api); + pi.entries.push(entry('e1', null)); + + await expect(pi.emit('message_end')).rejects.toThrow('store offline'); + }); + + it('routes failures to onError when the host wants to survive them', async () => { + const errors: unknown[] = []; + const store: RunLogAppendStore = { + append: async () => { + throw new Error('store offline'); + } + }; + const pi = fakePi(); + createRunLogExtension({ runId: 'run-1', store, onError: (error) => errors.push(error) }).extension(pi.api); + pi.entries.push(entry('e1', null)); + + await pi.emit('message_end'); + expect(errors).toHaveLength(1); + }); + + it('flushes on demand for a host that is shutting down', async () => { + const store = new MemoryRunLogStore(); + const pi = fakePi(); + const { extension, flush, mirror } = createRunLogExtension({ runId: 'run-1', store }); + extension(pi.api); + await pi.emit('session_start'); + + pi.entries.push(entry('e1', null)); + expect(await flush()).toHaveLength(1); + expect(await mirror.drain()).toEqual([]); + }); +}); diff --git a/agentic/pi-ext-run-log/__tests__/mirror.test.ts b/agentic/pi-ext-run-log/__tests__/mirror.test.ts new file mode 100644 index 0000000000..9b3437eb5c --- /dev/null +++ b/agentic/pi-ext-run-log/__tests__/mirror.test.ts @@ -0,0 +1,186 @@ +import { MemoryRunLogStore, projectParts, type RunLogAppendStore, START } from '@agentic-kit/run-log'; + +import { SessionMirror } from '../src/mirror'; + +interface FakeSession { + getHeader(): unknown; + getEntries(): readonly unknown[]; + push(entry: unknown): void; + reset(sessionId: string): void; +} + +const fakeSession = (sessionId = 'sess-1'): FakeSession => { + let header: Record = { type: 'session', version: 3, id: sessionId, timestamp: '2026-01-01T00:00:00.000Z', cwd: '/w' }; + let entries: unknown[] = []; + return { + getHeader: () => header, + getEntries: () => entries, + push: (entry) => { + entries.push(entry); + }, + reset: (id) => { + header = { ...header, id }; + entries = []; + } + }; +}; + +let parent: string | null = null; +let n = 0; +const message = (role: 'user' | 'assistant', text: string) => { + n += 1; + const id = `e${n}`; + const entry = { + type: 'message', + id, + parentId: parent, + timestamp: `2026-01-01T00:00:0${n}.000Z`, + message: { role, content: [{ type: 'text', text }] } + }; + parent = id; + return entry; +}; + +beforeEach(() => { + parent = null; + n = 0; +}); + +describe('SessionMirror', () => { + it('mirrors the header once, then every new entry, in order', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + + session.push(message('user', 'hi')); + expect(await mirror.drain()).toHaveLength(2); // header + entry + session.push(message('assistant', 'hello')); + expect(await mirror.drain()).toHaveLength(1); + expect(await mirror.drain()).toHaveLength(0); + + const page = await store.read('run-1', START); + expect(page.records.map((r) => r.seq)).toEqual([1, 2, 3]); + expect(page.records[0].entry.type).toBe('session'); + expect(projectParts(page.records).parts.map((p) => p.kind)).toEqual(['text', 'text']); + }); + + it('stores entries verbatim, including types it does not understand', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + + const exotic = { type: 'future_thing', id: 'x1', parentId: null as string | null, timestamp: '2026-01-01T00:00:01.000Z', payload: { deep: [1, 2] } }; + session.push(exotic); + await mirror.drain(); + + const page = await store.read('run-1', START); + expect(page.records[1].entry).toEqual(exotic); + }); + + it('does nothing until bound', async () => { + const store = new MemoryRunLogStore(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + expect(await mirror.drain()).toEqual([]); + }); + + it('serializes concurrent drains rather than interleaving batches', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + + session.push(message('user', 'a')); + const first = mirror.drain(); + session.push(message('assistant', 'b')); + const second = mirror.drain(); + const [a, b] = await Promise.all([first, second]); + + const seqs = [...a, ...b].map((r) => r.seq); + expect(seqs).toEqual([...seqs].sort((x, y) => x - y)); + const page = await store.read('run-1', START); + expect(page.records).toHaveLength(3); + }); + + it('retries the whole batch when an append fails, losing nothing', async () => { + const inner = new MemoryRunLogStore(); + let fail = true; + const store: RunLogAppendStore = { + append: async (runId, entries, options) => { + if (fail) { + fail = false; + throw new Error('transport down'); + } + return inner.append(runId, entries, options); + } + }; + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + + session.push(message('user', 'a')); + await expect(mirror.drain()).rejects.toThrow('transport down'); + session.push(message('assistant', 'b')); + expect(await mirror.drain()).toHaveLength(3); + expect((await inner.read('run-1', START)).records).toHaveLength(3); + }); + + it('re-mirrors from the start when pi switches to another session', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession('sess-1'); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + + session.push(message('user', 'a')); + await mirror.drain(); + + session.reset('sess-2'); + session.push(message('user', 'a-forked')); + await mirror.drain(); + + const page = await store.read('run-1', START); + const headers = page.records.filter((r) => r.entry.type === 'session'); + expect(headers).toHaveLength(2); + expect(page.records.map((r) => r.seq)).toEqual([1, 2, 3, 4]); + }); + + it('replays a resumed session idempotently', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + session.push(message('user', 'a')); + session.push(message('assistant', 'b')); + + const first = new SessionMirror({ runId: 'run-1', store }); + first.bind(session); + await first.drain(); + + // Fresh process, same session file, same run: nothing is written twice. + const resumed = new SessionMirror({ runId: 'run-1', store }); + resumed.bind(session); + expect(await resumed.drain()).toEqual([]); + expect((await store.read('run-1', START)).records).toHaveLength(3); + }); + + it('rejects a malformed entry loudly', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store }); + mirror.bind(session); + session.push({ type: 'message', message: {} }); + + await expect(mirror.drain()).rejects.toThrow(/id/); + }); + + it('passes the pi session version through to the records', async () => { + const store = new MemoryRunLogStore(); + const session = fakeSession(); + const mirror = new SessionMirror({ runId: 'run-1', store, piSessionVersion: 3 }); + mirror.bind(session); + session.push(message('user', 'a')); + await mirror.drain(); + + const page = await store.read('run-1', START); + expect(page.records.every((r) => r.piSessionVersion === 3)).toBe(true); + }); +}); diff --git a/agentic/pi-ext-run-log/jest.config.js b/agentic/pi-ext-run-log/jest.config.js new file mode 100644 index 0000000000..8a26efd6d0 --- /dev/null +++ b/agentic/pi-ext-run-log/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/pi-ext-run-log/package.json b/agentic/pi-ext-run-log/package.json new file mode 100644 index 0000000000..532ac6d800 --- /dev/null +++ b/agentic/pi-ext-run-log/package.json @@ -0,0 +1,47 @@ +{ + "name": "@agentic-kit/pi-ext-run-log", + "version": "0.1.0", + "author": "Constructive ", + "description": "pi extension that mirrors every session entry into an @agentic-kit/run-log store, verbatim and in order", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "dependencies": { + "@agentic-kit/run-log": "workspace:^" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": ">=0.79.0" + }, + "devDependencies": { + "@earendil-works/pi-coding-agent": "0.79.6" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "run-log", + "constructive" + ] +} diff --git a/agentic/pi-ext-run-log/src/extension.ts b/agentic/pi-ext-run-log/src/extension.ts new file mode 100644 index 0000000000..2f287ab284 --- /dev/null +++ b/agentic/pi-ext-run-log/src/extension.ts @@ -0,0 +1,89 @@ +/** + * The pi extension: drain the session mirror after anything that can append an + * entry. Every listed event is a point where pi has just written to the session, + * so the run log trails the session by at most one event. + */ + +import type { RunEventRecord, RunLogAppendStore } from '@agentic-kit/run-log'; +import type { ExtensionAPI, ExtensionFactory } from '@earendil-works/pi-coding-agent'; + +import { type SessionEntrySource, SessionMirror } from './mirror'; + +/** + * Events after which the session may have grown. `session_start` also covers + * resume (pi replays the file before the first event), so a resumed run + * re-appends its history and the store's idempotency discards the duplicates. + */ +export const MIRROR_EVENTS = [ + 'session_start', + 'session_compact', + 'session_tree', + 'session_shutdown', + 'input', + 'before_agent_start', + 'message_end', + 'tool_execution_end', + 'turn_end', + 'agent_end', + 'model_select', + 'thinking_level_select' +] as const; + +export type MirrorEvent = (typeof MIRROR_EVENTS)[number]; + +export interface RunLogExtensionOptions { + /** The run this session belongs to. Local and cloud runs differ only here. */ + runId: string; + store: RunLogAppendStore; + piSessionVersion?: number; + events?: readonly MirrorEvent[]; + /** + * Called when a drain fails. Without it the failure is rethrown into pi's + * event dispatch: a run log that silently stops recording is worse than a + * loud one, so losing entries is never the default. + */ + onError?: (error: unknown) => void; +} + +export interface RunLogExtension { + extension: ExtensionFactory; + /** Drain now — for a host that wants the log flushed before it exits. */ + flush(): Promise; + mirror: SessionMirror; +} + +export function createRunLogExtension(options: RunLogExtensionOptions): RunLogExtension { + const mirror = new SessionMirror({ + runId: options.runId, + store: options.store, + ...(options.piSessionVersion === undefined ? {} : { piSessionVersion: options.piSessionVersion }) + }); + const events = options.events ?? MIRROR_EVENTS; + + const drain = async (): Promise => { + try { + return await mirror.drain(); + } catch (error) { + if (!options.onError) throw error; + options.onError(error); + return []; + } + }; + + const extension: ExtensionFactory = (pi: ExtensionAPI) => { + for (const event of events) { + // Each overload of `on` is typed for its own handler; the handler here + // ignores the event and only reads the context, so one cast at the + // registration boundary keeps the loop. + (pi.on as (name: MirrorEvent, handler: (event: unknown, ctx: { sessionManager: unknown }) => Promise) => void)( + event, + async (_event, ctx) => { + mirror.bind(ctx.sessionManager as SessionEntrySource); + await drain(); + } + ); + } + }; + + return { extension, flush: drain, mirror }; +} diff --git a/agentic/pi-ext-run-log/src/index.ts b/agentic/pi-ext-run-log/src/index.ts new file mode 100644 index 0000000000..aefc3fbc3e --- /dev/null +++ b/agentic/pi-ext-run-log/src/index.ts @@ -0,0 +1,14 @@ +/** + * `@agentic-kit/pi-ext-run-log` — the write side of the run log for a pi + * session. Same extension locally and in the cloud; only `runId` and the store + * differ. + */ + +export { + createRunLogExtension, + MIRROR_EVENTS, + type MirrorEvent, + type RunLogExtension, + type RunLogExtensionOptions +} from './extension'; +export { type SessionEntrySource, SessionMirror, type SessionMirrorOptions } from './mirror'; diff --git a/agentic/pi-ext-run-log/src/mirror.ts b/agentic/pi-ext-run-log/src/mirror.ts new file mode 100644 index 0000000000..d1819da4fc --- /dev/null +++ b/agentic/pi-ext-run-log/src/mirror.ts @@ -0,0 +1,107 @@ +/** + * Mirroring pi's session into the run log. + * + * pi owns the session: it appends entries to an in-memory, append-only tree and + * (when persisted) a JSONL file. There is no "entry appended" event to subscribe + * to, so the mirror drains instead — after anything that could have appended, it + * takes the entries it has not seen yet and appends them to the run log + * verbatim. Index-based draining is sound precisely because the session is + * append-only: entries are never rewritten or removed, only branched from. + * + * A switch/fork/new-session replaces the entry list under the same manager, so + * the read position is keyed to the session header's id and resets when that id + * changes; entries carried into the new session are absorbed by the store's + * idempotency (pi entry ids). + * + * This file knows nothing about pi's extension API so it can be tested without a + * running agent; `./extension.ts` wires it to the events. + */ + +import { assertPiSessionEntry, type PiSessionEntry, type RunEventRecord, type RunLogAppendStore } from '@agentic-kit/run-log'; + +/** The slice of pi's `ReadonlySessionManager` the mirror needs. */ +export interface SessionEntrySource { + getHeader(): unknown; + getEntries(): readonly unknown[]; +} + +export interface SessionMirrorOptions { + runId: string; + store: RunLogAppendStore; + /** pi session format version the entries are produced under. */ + piSessionVersion?: number; +} + +export class SessionMirror { + private readonly runId: string; + private readonly store: RunLogAppendStore; + private readonly piSessionVersion: number | undefined; + + private source: SessionEntrySource | null = null; + private sessionId: string | null = null; + private consumed = 0; + private headerMirrored = false; + private tail: Promise = Promise.resolve(); + + constructor(options: SessionMirrorOptions) { + this.runId = options.runId; + this.store = options.store; + this.piSessionVersion = options.piSessionVersion; + } + + /** Point the mirror at a session source. Safe to call on every event. */ + bind(source: SessionEntrySource): void { + this.source = source; + } + + /** + * Append everything the mirror has not seen yet. Drains are serialized, so + * concurrent callers cannot interleave batches and break run-log ordering. + */ + drain(): Promise { + const run = this.tail.then(() => this.flushOnce()); + this.tail = run.then( + (): void => undefined, + (): void => undefined + ); + return run; + } + + private async flushOnce(): Promise { + const source = this.source; + if (!source) return []; + + const header = source.getHeader(); + const sessionId = headerSessionId(header); + if (sessionId !== null && sessionId !== this.sessionId) { + this.sessionId = sessionId; + this.consumed = 0; + this.headerMirrored = false; + } + + const batch: PiSessionEntry[] = []; + if (!this.headerMirrored && header !== null && header !== undefined) batch.push(assertPiSessionEntry(header)); + + const entries = source.getEntries(); + const upto = entries.length; + for (let i = this.consumed; i < upto; i += 1) batch.push(assertPiSessionEntry(entries[i])); + if (batch.length === 0) return []; + + const written = await this.store.append( + this.runId, + batch, + this.piSessionVersion === undefined ? undefined : { piSessionVersion: this.piSessionVersion } + ); + + // Advance only after a successful append: a failed drain is retried whole. + this.headerMirrored = true; + this.consumed = upto; + return written; + } +} + +function headerSessionId(header: unknown): string | null { + if (typeof header !== 'object' || header === null) return null; + const id = (header as { id?: unknown }).id; + return typeof id === 'string' && id.length > 0 ? id : null; +} diff --git a/agentic/pi-ext-run-log/tsconfig.esm.json b/agentic/pi-ext-run-log/tsconfig.esm.json new file mode 100644 index 0000000000..624ab17cfa --- /dev/null +++ b/agentic/pi-ext-run-log/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/pi-ext-run-log/tsconfig.json b/agentic/pi-ext-run-log/tsconfig.json new file mode 100644 index 0000000000..df063b5ee3 --- /dev/null +++ b/agentic/pi-ext-run-log/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c266a1191e..8f9cee2761 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,17 @@ importers: agentic/protocol: publishDirectory: dist + agentic/pi-ext-run-log: + dependencies: + '@agentic-kit/run-log': + specifier: workspace:^ + version: link:../run-log/dist + devDependencies: + '@earendil-works/pi-coding-agent': + specifier: 0.79.6 + version: 0.79.6(ws@8.20.1)(zod@4.4.3) + publishDirectory: dist + agentic/react: dependencies: '@agentic-kit/agent': From 21aa1a287359feac180d4dd968b2427664994dc7 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 14 Aug 2026 04:07:52 +0000 Subject: [PATCH 3/5] feat(pi-ext-metered-model): route pi model calls through the metered gateway --- .github/workflows/run-tests.yaml | 2 +- agentic/pi-ext-metered-model/README.md | 58 +++++++ .../__tests__/extension.test.ts | 125 ++++++++++++++ .../__tests__/provider.test.ts | 156 ++++++++++++++++++ agentic/pi-ext-metered-model/jest.config.js | 21 +++ agentic/pi-ext-metered-model/package.json | 45 +++++ agentic/pi-ext-metered-model/src/extension.ts | 59 +++++++ agentic/pi-ext-metered-model/src/identity.ts | 52 ++++++ agentic/pi-ext-metered-model/src/index.ts | 21 +++ agentic/pi-ext-metered-model/src/provider.ts | 117 +++++++++++++ .../pi-ext-metered-model/tsconfig.esm.json | 7 + agentic/pi-ext-metered-model/tsconfig.json | 8 + pnpm-lock.yaml | 7 + 13 files changed, 677 insertions(+), 1 deletion(-) create mode 100644 agentic/pi-ext-metered-model/README.md create mode 100644 agentic/pi-ext-metered-model/__tests__/extension.test.ts create mode 100644 agentic/pi-ext-metered-model/__tests__/provider.test.ts create mode 100644 agentic/pi-ext-metered-model/jest.config.js create mode 100644 agentic/pi-ext-metered-model/package.json create mode 100644 agentic/pi-ext-metered-model/src/extension.ts create mode 100644 agentic/pi-ext-metered-model/src/identity.ts create mode 100644 agentic/pi-ext-metered-model/src/index.ts create mode 100644 agentic/pi-ext-metered-model/src/provider.ts create mode 100644 agentic/pi-ext-metered-model/tsconfig.esm.json create mode 100644 agentic/pi-ext-metered-model/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 30ee5aaef4..5adb298669 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log agentic/pi-ext-metered-model' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/pi-ext-metered-model/README.md b/agentic/pi-ext-metered-model/README.md new file mode 100644 index 0000000000..94868c8326 --- /dev/null +++ b/agentic/pi-ext-metered-model/README.md @@ -0,0 +1,58 @@ +

+ +

+ +# @agentic-kit/pi-ext-metered-model + +A pi extension that makes the [Constructive metered gateway](../agentic-server) the session's model provider, so a coding-agent run is metered exactly like every other platform inference call — same `inference_log`, same identity, same billing. + +## Why the gateway and not self-reporting + +Two lanes exist for getting a pi run's usage into billing: + +| lane | package | authority | +| --- | --- | --- | +| model calls leave through `agentic-server` | **this package** | the gateway — the agent cannot under-report | +| the host owns the provider keys and reports usage afterwards | `@agentic-kit/pi-ext-usage-report` | self-reported; good for reconciliation, not for billing | + +Use this one for cloud runs. `agentic-server` already speaks OpenAI's `/v1/chat/completions`, which is one of pi's built-in api types, so no custom streaming code is involved: the extension registers a provider whose `baseUrl` is the gateway and whose headers carry the run's identity. + +## Usage + +```ts +import { createMeteredModelExtension } from '@agentic-kit/pi-ext-metered-model'; + +const metered = createMeteredModelExtension({ + gatewayUrl: 'https://agentic.example.com', // the gateway root, NOT the /v1 path + identity: { + databaseId: process.env.CONSTRUCTIVE_DATABASE_ID!, + entityId: process.env.CONSTRUCTIVE_OWNER_ID, + actorId: runActorId, + runToken // run-scoped, never an account token + }, + models: [ + { id: 'anthropic/claude-sonnet-4', contextWindow: 200000, maxTokens: 8192, input: ['text', 'image'] } + ] +}); + +// hand `metered.extension` to pi alongside your other extensions +``` + +The first declared model is selected on `session_start`; pass `selectModel: ''` to pick another, or `selectModel: false` to leave the host's choice alone. + +## Behavior + +- **Identity** travels as `X-Database-Id` / `X-Entity-Id` / `X-Actor-Id`, the headers `agentic-server` already reads, plus `Authorization: Bearer ` when one is given. Extra `headers` (e.g. `X-LLM-Provider`) are merged, but can never shadow identity. +- **Headers are only as trustworthy as the network.** In-cluster, or behind an ingress that pins identity from the bearer, they are authoritative; a host outside that boundary must send `runToken` and let the ingress do the pinning. +- **Misconfiguration fails at construction**, not at the first turn: a blank `databaseId`, a relative or non-http `gatewayUrl`, a `gatewayUrl` ending in `/v1` (pi appends its own, so the request would 404), an empty model list, or a `selectModel` that is not one of the registered models — the last of which would otherwise leave pi quietly running an *unmetered* model. +- **Model selection fails loudly** if the model is absent from the registry after registration or pi refuses it for lack of credentials. +- **Cost fields default to zero.** The gateway is the pricing authority; zeros mean "not priced client-side", not free. Pass `cost` when you want pi's UI to show numbers. + +## Testing + +`pnpm test` — the provider/identity builders are asserted directly, and the extension is driven through a fake `ExtensionAPI`, so no gateway or model credentials are needed. + +## Related + +- `@agentic-kit/run-log` — append-only run log +- `@agentic-kit/pi-ext-run-log` — mirrors pi session entries into it diff --git a/agentic/pi-ext-metered-model/__tests__/extension.test.ts b/agentic/pi-ext-metered-model/__tests__/extension.test.ts new file mode 100644 index 0000000000..c3867713f9 --- /dev/null +++ b/agentic/pi-ext-metered-model/__tests__/extension.test.ts @@ -0,0 +1,125 @@ +import type { ExtensionAPI, ProviderConfig } from '@earendil-works/pi-coding-agent'; + +import { createMeteredModelExtension } from '../src'; + +const models = [ + { id: 'anthropic/claude-sonnet-4', contextWindow: 200000, maxTokens: 8192 }, + { id: 'openai/gpt-5', contextWindow: 400000, maxTokens: 16384 } +]; + +const options = { + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'db-1', runToken: 'tok-1' }, + models +}; + +interface FakePi { + api: ExtensionAPI; + providers: { name: string; config: ProviderConfig }[]; + handlers: Map Promise | void>; + setModel: jest.Mock; +} + +function fakePi(modelAccepted = true): FakePi { + const providers: FakePi['providers'] = []; + const handlers = new Map Promise | void>(); + const setModel = jest.fn().mockResolvedValue(modelAccepted); + const api = { + registerProvider: (name: string, config: ProviderConfig) => providers.push({ name, config }), + on: (event: string, handler: (event: unknown, ctx: unknown) => Promise | void) => handlers.set(event, handler), + setModel + } as unknown as ExtensionAPI; + return { api, providers, handlers, setModel }; +} + +function fakeCtx(found: unknown) { + const find = jest.fn().mockReturnValue(found); + return { ctx: { modelRegistry: { find } } as never, find }; +} + +describe('createMeteredModelExtension', () => { + it('registers the gateway provider under the default name', () => { + const pi = fakePi(); + createMeteredModelExtension(options).extension(pi.api); + + expect(pi.providers).toHaveLength(1); + expect(pi.providers[0].name).toBe('constructive-gateway'); + expect(pi.providers[0].config.baseUrl).toBe('https://agentic.example.com'); + }); + + it('honours a custom provider name', () => { + const pi = fakePi(); + const ext = createMeteredModelExtension({ ...options, providerName: 'tenant-gateway' }); + ext.extension(pi.api); + + expect(pi.providers[0].name).toBe('tenant-gateway'); + expect(ext.providerName).toBe('tenant-gateway'); + }); + + it('selects the first model on session start', async () => { + const pi = fakePi(); + const ext = createMeteredModelExtension(options); + ext.extension(pi.api); + + expect(ext.selectedModel).toBe('anthropic/claude-sonnet-4'); + + const model = { id: 'anthropic/claude-sonnet-4' }; + const { ctx, find } = fakeCtx(model); + await pi.handlers.get('session_start')!({ type: 'session_start' }, ctx); + + expect(find).toHaveBeenCalledWith('constructive-gateway', 'anthropic/claude-sonnet-4'); + expect(pi.setModel).toHaveBeenCalledWith(model); + }); + + it('selects an explicitly requested model', async () => { + const pi = fakePi(); + const ext = createMeteredModelExtension({ ...options, selectModel: 'openai/gpt-5' }); + ext.extension(pi.api); + + const { ctx, find } = fakeCtx({ id: 'openai/gpt-5' }); + await pi.handlers.get('session_start')!({ type: 'session_start' }, ctx); + + expect(find).toHaveBeenCalledWith('constructive-gateway', 'openai/gpt-5'); + expect(ext.selectedModel).toBe('openai/gpt-5'); + }); + + it('rejects selecting a model it does not register, which would silently stay unmetered', () => { + expect(() => createMeteredModelExtension({ ...options, selectModel: 'anthropic/other' })).toThrow( + /not one of the registered models/ + ); + }); + + it('leaves the host model choice alone when selection is disabled', () => { + const pi = fakePi(); + const ext = createMeteredModelExtension({ ...options, selectModel: false }); + ext.extension(pi.api); + + expect(ext.selectedModel).toBeUndefined(); + expect(pi.handlers.size).toBe(0); + expect(pi.providers).toHaveLength(1); + }); + + it('fails loudly when the model is missing after registration', async () => { + const pi = fakePi(); + createMeteredModelExtension(options).extension(pi.api); + + const { ctx } = fakeCtx(undefined); + await expect(pi.handlers.get('session_start')!({ type: 'session_start' }, ctx)).rejects.toThrow(/has no model/); + }); + + it('fails loudly when pi refuses the model for lack of credentials', async () => { + const pi = fakePi(false); + createMeteredModelExtension(options).extension(pi.api); + + const { ctx } = fakeCtx({ id: 'anthropic/claude-sonnet-4' }); + await expect(pi.handlers.get('session_start')!({ type: 'session_start' }, ctx)).rejects.toThrow( + /refused model .*no usable credentials/ + ); + }); + + it('surfaces identity misconfiguration before any turn runs', () => { + expect(() => createMeteredModelExtension({ ...options, identity: { databaseId: '' } })).toThrow( + /databaseId is required/ + ); + }); +}); diff --git a/agentic/pi-ext-metered-model/__tests__/provider.test.ts b/agentic/pi-ext-metered-model/__tests__/provider.test.ts new file mode 100644 index 0000000000..fe0a6f8cbb --- /dev/null +++ b/agentic/pi-ext-metered-model/__tests__/provider.test.ts @@ -0,0 +1,156 @@ +import { + ACTOR_ID_HEADER, + buildIdentityHeaders, + DATABASE_ID_HEADER, + ENTITY_ID_HEADER, + GATEWAY_API, + meteredModelConfig, + meteredProviderConfig, + normalizeGatewayUrl +} from '../src'; + +const models = [{ id: 'anthropic/claude-sonnet-4', contextWindow: 200000, maxTokens: 8192 }]; + +describe('buildIdentityHeaders', () => { + it('sends the gateway identity headers', () => { + expect( + buildIdentityHeaders({ + databaseId: 'db-1', + entityId: 'ent-1', + actorId: 'actor-1', + runToken: 'tok-1' + }) + ).toEqual({ + [DATABASE_ID_HEADER]: 'db-1', + [ENTITY_ID_HEADER]: 'ent-1', + [ACTOR_ID_HEADER]: 'actor-1', + Authorization: 'Bearer tok-1' + }); + }); + + it('omits optional identity rather than sending blanks', () => { + expect(buildIdentityHeaders({ databaseId: 'db-1', entityId: ' ', actorId: '' })).toEqual({ + [DATABASE_ID_HEADER]: 'db-1' + }); + }); + + it('rejects a missing databaseId up front', () => { + expect(() => buildIdentityHeaders({ databaseId: ' ' })).toThrow(/databaseId is required/); + }); +}); + +describe('normalizeGatewayUrl', () => { + it('keeps the origin and strips trailing slashes', () => { + expect(normalizeGatewayUrl('https://agentic.example.com/')).toBe('https://agentic.example.com'); + expect(normalizeGatewayUrl(' http://localhost:3000 ')).toBe('http://localhost:3000'); + }); + + it('preserves a mount path', () => { + expect(normalizeGatewayUrl('https://example.com/gateway/')).toBe('https://example.com/gateway'); + }); + + it('rejects a /v1 suffix, which pi would double up', () => { + expect(() => normalizeGatewayUrl('https://example.com/v1')).toThrow(/drop the \/v1/); + }); + + it('rejects relative and non-http urls', () => { + expect(() => normalizeGatewayUrl('/v1/chat')).toThrow(/absolute URL/); + expect(() => normalizeGatewayUrl('ws://example.com')).toThrow(/http\(s\)/); + expect(() => normalizeGatewayUrl('')).toThrow(/gatewayUrl is required/); + }); +}); + +describe('meteredModelConfig', () => { + it('fills the model fields pi requires with safe defaults', () => { + expect(meteredModelConfig(models[0])).toEqual({ + id: 'anthropic/claude-sonnet-4', + name: 'anthropic/claude-sonnet-4', + reasoning: false, + input: ['text'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 8192 + }); + }); + + it('passes through declared capabilities and cost', () => { + expect( + meteredModelConfig({ + id: 'm', + name: 'M', + contextWindow: 1, + maxTokens: 2, + reasoning: true, + input: ['text', 'image'], + cost: { input: 3, output: 4, cacheRead: 5, cacheWrite: 6 } + }) + ).toMatchObject({ name: 'M', reasoning: true, input: ['text', 'image'], cost: { input: 3, cacheWrite: 6 } }); + }); + + it('rejects a blank model id', () => { + expect(() => meteredModelConfig({ id: ' ', contextWindow: 1, maxTokens: 1 })).toThrow(/model id is required/); + }); +}); + +describe('meteredProviderConfig', () => { + it('points pi at the gateway over the openai-completions api', () => { + const config = meteredProviderConfig({ + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'db-1' }, + models + }); + + expect(config.baseUrl).toBe('https://agentic.example.com'); + expect(config.api).toBe(GATEWAY_API); + expect(config.models).toHaveLength(1); + expect(config.headers?.[DATABASE_ID_HEADER]).toBe('db-1'); + }); + + it('merges extra headers but never lets them shadow identity', () => { + const config = meteredProviderConfig({ + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'real-db' }, + models, + headers: { 'X-LLM-Provider': 'anthropic', [DATABASE_ID_HEADER]: 'spoofed' } + }); + + expect(config.headers).toMatchObject({ 'X-LLM-Provider': 'anthropic', [DATABASE_ID_HEADER]: 'real-db' }); + }); + + it('uses the run token as the api key so pi passes its auth check', () => { + const config = meteredProviderConfig({ + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'db-1', runToken: 'tok-1' }, + models + }); + + expect(config.apiKey).toBe('tok-1'); + }); + + it('falls back to a placeholder api key when identity travels in headers only', () => { + const config = meteredProviderConfig({ + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'db-1' }, + models + }); + + expect(config.apiKey).toBe('unused'); + }); + + it('prefers an explicit api key', () => { + const config = meteredProviderConfig({ + gatewayUrl: 'https://agentic.example.com', + identity: { databaseId: 'db-1', runToken: 'tok-1' }, + models, + apiKey: 'explicit' + }); + + expect(config.apiKey).toBe('explicit'); + }); + + it('requires at least one model', () => { + expect(() => + meteredProviderConfig({ gatewayUrl: 'https://agentic.example.com', identity: { databaseId: 'db-1' }, models: [] }) + ).toThrow(/at least one model/); + }); +}); diff --git a/agentic/pi-ext-metered-model/jest.config.js b/agentic/pi-ext-metered-model/jest.config.js new file mode 100644 index 0000000000..8a26efd6d0 --- /dev/null +++ b/agentic/pi-ext-metered-model/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/pi-ext-metered-model/package.json b/agentic/pi-ext-metered-model/package.json new file mode 100644 index 0000000000..f0e3d34e51 --- /dev/null +++ b/agentic/pi-ext-metered-model/package.json @@ -0,0 +1,45 @@ +{ + "name": "@agentic-kit/pi-ext-metered-model", + "version": "0.1.0", + "author": "Constructive ", + "description": "pi extension that routes model calls through the Constructive metered gateway, so a pi session is metered exactly like agentic-server traffic", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": ">=0.79.0" + }, + "devDependencies": { + "@earendil-works/pi-coding-agent": "0.79.6" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "metering", + "billing", + "constructive" + ] +} diff --git a/agentic/pi-ext-metered-model/src/extension.ts b/agentic/pi-ext-metered-model/src/extension.ts new file mode 100644 index 0000000000..48deeca14c --- /dev/null +++ b/agentic/pi-ext-metered-model/src/extension.ts @@ -0,0 +1,59 @@ +/** + * The pi extension: register the gateway as a provider, optionally as the + * session's model. + * + * This is the cloud metering lane — every model call leaves the process through + * `agentic-server`, which is the billing authority, so usage cannot be + * under-reported by the agent. The local lane (own provider keys, self-reported + * usage) is `@agentic-kit/pi-ext-usage-report`. + */ + +import type { ExtensionAPI, ExtensionFactory, ProviderConfig } from '@earendil-works/pi-coding-agent'; + +import { DEFAULT_PROVIDER_NAME, meteredProviderConfig,type MeteredProviderOptions } from './provider'; + +export interface MeteredModelExtensionOptions extends MeteredProviderOptions { + /** + * Model id to select once registered. Defaults to the first model; pass `false` + * to leave the host's model choice alone. + */ + selectModel?: string | false; +} + +export interface MeteredModelExtension { + extension: ExtensionFactory; + /** The config handed to pi — asserted in tests, useful for host logging. */ + config: ProviderConfig; + providerName: string; + /** Model id pi selects on session start, if any. */ + selectedModel: string | undefined; +} + +export function createMeteredModelExtension(options: MeteredModelExtensionOptions): MeteredModelExtension { + const providerName = options.providerName ?? DEFAULT_PROVIDER_NAME; + const config = meteredProviderConfig(options); + + const modelId = options.selectModel === false ? undefined : (options.selectModel ?? options.models[0].id); + if (modelId !== undefined && !options.models.some((model) => model.id === modelId)) { + // Selecting an unregistered id silently leaves pi on an unmetered model, + // which is the one failure this package exists to prevent. + throw new Error(`metered model: selectModel "${modelId}" is not one of the registered models`); + } + + const extension: ExtensionFactory = (pi: ExtensionAPI) => { + pi.registerProvider(providerName, config); + if (modelId === undefined) return; + + // `pi.setModel` takes a resolved model, and the registry that resolves it + // lives on the context — which an extension only gets on an event. + // `session_start` is the first, and fires before the first turn. + pi.on('session_start', async (_event, ctx) => { + const model = ctx.modelRegistry.find(providerName, modelId); + if (!model) throw new Error(`metered model: provider "${providerName}" has no model "${modelId}" after registration`); + const ok = await pi.setModel(model); + if (!ok) throw new Error(`metered model: pi refused model "${providerName}/${modelId}" (no usable credentials for the gateway)`); + }); + }; + + return { extension, config, providerName, selectedModel: modelId }; +} diff --git a/agentic/pi-ext-metered-model/src/identity.ts b/agentic/pi-ext-metered-model/src/identity.ts new file mode 100644 index 0000000000..0aab508faf --- /dev/null +++ b/agentic/pi-ext-metered-model/src/identity.ts @@ -0,0 +1,52 @@ +/** + * Who a metered request belongs to. + * + * These are exactly the headers `agentic-server` reads (`X-Database-Id`, + * `X-Entity-Id`, `X-Actor-Id`) — the same identity lane the platform's own + * clients use — so a pi session lands in `inference_log` beside every other + * metered call rather than in a parallel accounting scheme. + * + * Headers are only trustworthy where the gateway is not reachable by the + * untrusted party: in-cluster, or behind an authenticated ingress that pins the + * identity from the bearer. Off-cluster hosts must send `runToken` and let the + * ingress do the pinning. + */ + +export interface MeteredIdentity { + /** Tenant database the usage is billed to. Required by the gateway. */ + databaseId: string; + /** Owning entity (organization/user) when the platform tracks one. */ + entityId?: string; + /** The actor on whose behalf the run executes. */ + actorId?: string; + /** + * Bearer token for the gateway's ingress — run-scoped, not an account token. + * Sent as `Authorization: Bearer `. + */ + runToken?: string; +} + +export const DATABASE_ID_HEADER = 'X-Database-Id'; +export const ENTITY_ID_HEADER = 'X-Entity-Id'; +export const ACTOR_ID_HEADER = 'X-Actor-Id'; + +/** + * Build the identity headers for a metered request. + * + * Throws on a missing/blank `databaseId`: the gateway would reject the call with + * a 400 at the first model turn, which surfaces as an opaque agent failure much + * later than the misconfiguration. + */ +export function buildIdentityHeaders(identity: MeteredIdentity): Record { + const databaseId = identity.databaseId?.trim(); + if (!databaseId) throw new Error('metered model: identity.databaseId is required'); + + const headers: Record = { [DATABASE_ID_HEADER]: databaseId }; + const entityId = identity.entityId?.trim(); + if (entityId) headers[ENTITY_ID_HEADER] = entityId; + const actorId = identity.actorId?.trim(); + if (actorId) headers[ACTOR_ID_HEADER] = actorId; + const runToken = identity.runToken?.trim(); + if (runToken) headers.Authorization = `Bearer ${runToken}`; + return headers; +} diff --git a/agentic/pi-ext-metered-model/src/index.ts b/agentic/pi-ext-metered-model/src/index.ts new file mode 100644 index 0000000000..54ae7800b0 --- /dev/null +++ b/agentic/pi-ext-metered-model/src/index.ts @@ -0,0 +1,21 @@ +/** + * `@agentic-kit/pi-ext-metered-model` — route a pi session's model calls through + * the Constructive metered gateway (`agentic-server`), so cloud runs are metered + * by the gateway rather than trusting the agent's own accounting. + */ + +export { + createMeteredModelExtension, + type MeteredModelExtension, + type MeteredModelExtensionOptions +} from './extension'; +export { ACTOR_ID_HEADER, buildIdentityHeaders, DATABASE_ID_HEADER, ENTITY_ID_HEADER, type MeteredIdentity } from './identity'; +export { + DEFAULT_PROVIDER_NAME, + GATEWAY_API, + meteredModelConfig, + type MeteredModelSpec, + meteredProviderConfig, + type MeteredProviderOptions, + normalizeGatewayUrl +} from './provider'; diff --git a/agentic/pi-ext-metered-model/src/provider.ts b/agentic/pi-ext-metered-model/src/provider.ts new file mode 100644 index 0000000000..6de2d40f12 --- /dev/null +++ b/agentic/pi-ext-metered-model/src/provider.ts @@ -0,0 +1,117 @@ +/** + * The gateway as a pi provider. + * + * `agentic-server` speaks OpenAI's `/v1/chat/completions`, which is one of pi's + * built-in api types, so routing a pi session through it needs no custom + * streaming code — only a provider whose `baseUrl` is the gateway and whose + * headers carry the run's identity. + * + * This module builds the config as a plain object so it can be asserted in tests + * without pi's registry; `./extension.ts` hands it to `pi.registerProvider`. + */ + +import type { ProviderConfig, ProviderModelConfig } from '@earendil-works/pi-coding-agent'; + +import { buildIdentityHeaders, type MeteredIdentity } from './identity'; + +/** The api the gateway exposes. */ +export const GATEWAY_API = 'openai-completions'; + +/** Default provider name; also the prefix pi shows in the model picker. */ +export const DEFAULT_PROVIDER_NAME = 'constructive-gateway'; + +export interface MeteredModelSpec { + /** Model id as the gateway routes it, e.g. `anthropic/claude-sonnet-4`. */ + id: string; + name?: string; + contextWindow: number; + maxTokens: number; + reasoning?: boolean; + input?: ('text' | 'image')[]; + /** + * Per-token cost for pi's own display. The gateway is the billing authority, + * so zeros here mean "not priced client-side", never "free". + */ + cost?: ProviderModelConfig['cost']; +} + +export interface MeteredProviderOptions { + /** Gateway root, e.g. `https://agentic.example.com` — not a `/v1` path. */ + gatewayUrl: string; + identity: MeteredIdentity; + models: readonly MeteredModelSpec[]; + /** Provider name to register under. Defaults to `constructive-gateway`. */ + providerName?: string; + /** Display name in pi's UI. */ + displayName?: string; + /** Extra headers, e.g. `X-LLM-Provider` to pin the upstream provider. */ + headers?: Record; + /** + * API key for the gateway itself. Identity travels in headers, so this is + * usually unnecessary; when the ingress wants a bearer, prefer + * `identity.runToken`. + */ + apiKey?: string; +} + +const ZERO_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; + +/** Normalize the gateway root: absolute http(s), no trailing slash, no `/v1`. */ +export function normalizeGatewayUrl(gatewayUrl: string): string { + const raw = gatewayUrl?.trim(); + if (!raw) throw new Error('metered model: gatewayUrl is required'); + + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + throw new Error(`metered model: gatewayUrl must be an absolute URL, got ${raw}`); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`metered model: gatewayUrl must be http(s), got ${parsed.protocol}`); + } + + const path = parsed.pathname.replace(/\/+$/, ''); + // pi appends `/v1/chat/completions`, so a baseUrl that already ends in `/v1` + // would request `/v1/v1/chat/completions` and 404 at the first model turn. + if (/\/v1$/.test(path)) throw new Error(`metered model: gatewayUrl must be the gateway root, not ${raw} (drop the /v1)`); + return `${parsed.origin}${path}`; +} + +export function meteredModelConfig(spec: MeteredModelSpec): ProviderModelConfig { + if (!spec.id?.trim()) throw new Error('metered model: model id is required'); + return { + id: spec.id, + name: spec.name ?? spec.id, + reasoning: spec.reasoning ?? false, + input: spec.input ? [...spec.input] : ['text'], + cost: spec.cost ?? { ...ZERO_COST }, + contextWindow: spec.contextWindow, + maxTokens: spec.maxTokens + }; +} + +/** + * Build the pi provider config for the gateway. + * + * Identity headers are computed once here rather than per request: a run's + * identity is fixed for its lifetime, and pi has no per-request header hook that + * a provider config participates in. + */ +export function meteredProviderConfig(options: MeteredProviderOptions): ProviderConfig { + if (options.models.length === 0) throw new Error('metered model: at least one model is required'); + + const headers = { ...options.headers, ...buildIdentityHeaders(options.identity) }; + const config: ProviderConfig = { + name: options.displayName ?? 'Constructive (metered)', + baseUrl: normalizeGatewayUrl(options.gatewayUrl), + api: GATEWAY_API, + headers, + models: options.models.map(meteredModelConfig) + }; + // pi requires an apiKey when models are declared; the gateway authenticates by + // identity headers, so a placeholder keeps its validation satisfied without + // implying a real secret. + config.apiKey = options.apiKey ?? headers.Authorization?.replace(/^Bearer /, '') ?? 'unused'; + return config; +} diff --git a/agentic/pi-ext-metered-model/tsconfig.esm.json b/agentic/pi-ext-metered-model/tsconfig.esm.json new file mode 100644 index 0000000000..624ab17cfa --- /dev/null +++ b/agentic/pi-ext-metered-model/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/pi-ext-metered-model/tsconfig.json b/agentic/pi-ext-metered-model/tsconfig.json new file mode 100644 index 0000000000..df063b5ee3 --- /dev/null +++ b/agentic/pi-ext-metered-model/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f9cee2761..567d265dd7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,13 @@ importers: agentic/protocol: publishDirectory: dist + agentic/pi-ext-metered-model: + devDependencies: + '@earendil-works/pi-coding-agent': + specifier: 0.79.6 + version: 0.79.6(ws@8.20.1)(zod@4.4.3) + publishDirectory: dist + agentic/pi-ext-run-log: dependencies: '@agentic-kit/run-log': From a71cae2ae42115d33b4a187656f87473e846a918 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 14 Aug 2026 04:21:09 +0000 Subject: [PATCH 4/5] feat(pi-ext-usage-report): report pi session usage to the gateway --- .github/workflows/run-tests.yaml | 2 +- agentic/pi-ext-usage-report/README.md | 54 +++++++ .../__tests__/extension.test.ts | 149 ++++++++++++++++++ .../__tests__/report.test.ts | 86 ++++++++++ .../__tests__/reporter.test.ts | 123 +++++++++++++++ agentic/pi-ext-usage-report/jest.config.js | 21 +++ agentic/pi-ext-usage-report/package.json | 48 ++++++ agentic/pi-ext-usage-report/src/extension.ts | 81 ++++++++++ agentic/pi-ext-usage-report/src/index.ts | 25 +++ agentic/pi-ext-usage-report/src/report.ts | 95 +++++++++++ agentic/pi-ext-usage-report/src/reporter.ts | 90 +++++++++++ agentic/pi-ext-usage-report/tsconfig.esm.json | 7 + agentic/pi-ext-usage-report/tsconfig.json | 8 + pnpm-lock.yaml | 11 ++ 14 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 agentic/pi-ext-usage-report/README.md create mode 100644 agentic/pi-ext-usage-report/__tests__/extension.test.ts create mode 100644 agentic/pi-ext-usage-report/__tests__/report.test.ts create mode 100644 agentic/pi-ext-usage-report/__tests__/reporter.test.ts create mode 100644 agentic/pi-ext-usage-report/jest.config.js create mode 100644 agentic/pi-ext-usage-report/package.json create mode 100644 agentic/pi-ext-usage-report/src/extension.ts create mode 100644 agentic/pi-ext-usage-report/src/index.ts create mode 100644 agentic/pi-ext-usage-report/src/report.ts create mode 100644 agentic/pi-ext-usage-report/src/reporter.ts create mode 100644 agentic/pi-ext-usage-report/tsconfig.esm.json create mode 100644 agentic/pi-ext-usage-report/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 5adb298669..a02731934c 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log agentic/pi-ext-metered-model' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log agentic/pi-ext-metered-model agentic/pi-ext-usage-report' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/pi-ext-usage-report/README.md b/agentic/pi-ext-usage-report/README.md new file mode 100644 index 0000000000..20896b6d05 --- /dev/null +++ b/agentic/pi-ext-usage-report/README.md @@ -0,0 +1,54 @@ +

+ +

+ +# @agentic-kit/pi-ext-usage-report + +A pi extension that reports each assistant message's token usage and cost to the [Constructive gateway](../agentic-server)'s `POST /v1/usage`, so a run on the host's own provider keys still lands in `inference_log`. + +## Which lane is this + +| lane | package | authority | +| --- | --- | --- | +| model calls leave through `agentic-server` | `@agentic-kit/pi-ext-metered-model` | the gateway — the agent cannot under-report | +| the host owns the provider keys and reports afterwards | **this package** | self-reported: usage visibility and reconciliation, not tamper-proof billing | + +Both write the same rows. If a run is billed, route it through the gateway; this package exists for local/desktop runs on a developer's own key, where there is no proxy to meter and the alternative is no record at all. Nothing here needs a gateway change — `/v1/usage` already exists. + +## Usage + +```ts +import { createUsageReportExtension } from '@agentic-kit/pi-ext-usage-report'; + +const usage = createUsageReportExtension({ + gatewayUrl: 'https://agentic.example.com', // the gateway root, NOT the /v1 path + identity: { + databaseId: process.env.CONSTRUCTIVE_DATABASE_ID!, + entityId: process.env.CONSTRUCTIVE_OWNER_ID, + actorId: runActorId, + runToken + } +}); + +// hand `usage.extension` to pi; call `usage.flush()` when the host shuts the session down +``` + +`sink` replaces HTTP delivery entirely (in-process metering, a queue, a test double), in which case `gatewayUrl` is unnecessary. + +## Behavior + +- **What is reported.** pi puts `{ input, output, cacheRead, cacheWrite, totalTokens, cost }` on every assistant message — richer than the gateway's own proxy path sees. `input_tokens` is every prompt token the provider processed (`input + cacheRead + cacheWrite`), because `input` alone excludes cache hits and would under-report a long agent session by an order of magnitude; the split and pi's cost breakdown survive verbatim in `raw_usage`. +- **A turn that never reached the provider reports nothing** — a row of zeros is worse than no row. +- **Failed turns are still reported**, with `status: 'error'` and the provider's message as `error_type`; those tokens were spent. +- **Double billing is prevented** by deduping on the provider's `responseId`, since pi can emit `message_end` more than once for one response (rewritten message, replay on resume). +- **Delivery never sits in the agent's turn latency.** Reports are queued and sent serially; the first failure is retained and rethrown from `flush()` (also called on `session_shutdown`), so a host that flushes still fails loudly instead of silently losing usage. Pass `onError` to make delivery failures non-fatal. + +## Testing + +`pnpm test` — report shaping, the HTTP sink (injected `fetch`), and queue/failure semantics are asserted directly; no gateway needed. + +## Related + +- `@agentic-kit/pi-ext-metered-model` — the authoritative cloud lane +- `@agentic-kit/run-log` — append-only run log +- `@agentic-kit/pi-ext-run-log` — mirrors pi session entries into it diff --git a/agentic/pi-ext-usage-report/__tests__/extension.test.ts b/agentic/pi-ext-usage-report/__tests__/extension.test.ts new file mode 100644 index 0000000000..ca76fca52b --- /dev/null +++ b/agentic/pi-ext-usage-report/__tests__/extension.test.ts @@ -0,0 +1,149 @@ +import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; + +import { createUsageReportExtension, type UsageReport } from '../src'; + +type Handler = (event: any, ctx: any) => unknown; + +const fakePi = () => { + const handlers = new Map(); + const pi = { + on: (event: string, handler: Handler) => { + handlers.set(event, handler); + } + } as unknown as ExtensionAPI; + return { + pi, + emit: (event: string, payload: unknown) => handlers.get(event)?.(payload, {}), + has: (event: string) => handlers.has(event) + }; +}; + +const assistant = (overrides: Record = {}) => ({ + role: 'assistant', + provider: 'openai', + model: 'gpt-4o', + stopReason: 'stop', + usage: { input: 10, output: 5, cacheRead: 0, cacheWrite: 0, totalTokens: 15 }, + ...overrides +}); + +describe('createUsageReportExtension', () => { + const identity = { databaseId: 'db-1' }; + + const setup = (options: Record = {}) => { + const reports: UsageReport[] = []; + const ext = createUsageReportExtension({ + identity, + sink: async (report) => { + reports.push(report); + }, + ...options + }); + const host = fakePi(); + ext.extension(host.pi); + return { ext, host, reports }; + }; + + it('reports the usage of each assistant message', async () => { + const { ext, host, reports } = setup(); + + host.emit('message_end', { type: 'message_end', message: assistant({ responseId: 'r1' }) }); + host.emit('message_end', { type: 'message_end', message: assistant({ responseId: 'r2', model: 'gpt-4o-mini' }) }); + await ext.flush(); + + expect(reports.map((r) => r.model)).toEqual(['gpt-4o', 'gpt-4o-mini']); + expect(reports[0]).toMatchObject({ provider: 'openai', input_tokens: 10, output_tokens: 5, status: 'ok' }); + }); + + it('ignores user and tool-result messages', async () => { + const { ext, host, reports } = setup(); + + host.emit('message_end', { type: 'message_end', message: { role: 'user', content: 'hi' } }); + host.emit('message_end', { type: 'message_end', message: { role: 'toolResult', toolName: 'bash' } }); + await ext.flush(); + + expect(reports).toEqual([]); + }); + + it('bills a response once even when pi ends the same message twice', async () => { + const { ext, host, reports } = setup(); + const message = assistant({ responseId: 'r1' }); + + host.emit('message_end', { type: 'message_end', message }); + host.emit('message_end', { type: 'message_end', message }); + await ext.flush(); + + expect(reports).toHaveLength(1); + }); + + it('still reports messages that carry no responseId', async () => { + const { ext, host, reports } = setup(); + + host.emit('message_end', { type: 'message_end', message: assistant() }); + host.emit('message_end', { type: 'message_end', message: assistant() }); + await ext.flush(); + + expect(reports).toHaveLength(2); + }); + + it('applies the configured operation label', async () => { + const { ext, host, reports } = setup({ operation: 'pi/code-task' }); + + host.emit('message_end', { type: 'message_end', message: assistant() }); + await ext.flush(); + + expect(reports[0].operation).toBe('pi/code-task'); + }); + + it('flushes on session_shutdown so a quitting host does not drop usage', async () => { + const reports: UsageReport[] = []; + let release = (): void => undefined; + const inFlight = new Promise((resolve) => { + release = resolve; + }); + const ext = createUsageReportExtension({ + identity, + sink: async (report) => { + await inFlight; + reports.push(report); + } + }); + const host = fakePi(); + ext.extension(host.pi); + + host.emit('message_end', { type: 'message_end', message: assistant() }); + const shutdown = host.emit('session_shutdown', { type: 'session_shutdown', reason: 'quit' }) as Promise; + expect(reports).toEqual([]); + + release(); + await shutdown; + expect(reports).toHaveLength(1); + }); + + it('surfaces delivery failures from the shutdown flush', async () => { + const ext = createUsageReportExtension({ + identity, + sink: () => Promise.reject(new Error('gateway down')) + }); + const host = fakePi(); + ext.extension(host.pi); + + host.emit('message_end', { type: 'message_end', message: assistant() }); + await expect(host.emit('session_shutdown', { type: 'session_shutdown', reason: 'quit' })).rejects.toThrow( + 'gateway down' + ); + }); + + it('requires a gatewayUrl when no sink is supplied', () => { + expect(() => createUsageReportExtension({ identity })).toThrow(/gatewayUrl is required/); + }); + + it('validates the gateway URL and identity up front, not on the first turn', () => { + expect(() => createUsageReportExtension({ identity, gatewayUrl: 'agentic.example.com' })).toThrow( + /absolute URL/ + ); + expect(() => + createUsageReportExtension({ identity: { databaseId: ' ' }, gatewayUrl: 'https://gw.example.com' }) + ).toThrow(/databaseId is required/); + }); +}); diff --git a/agentic/pi-ext-usage-report/__tests__/report.test.ts b/agentic/pi-ext-usage-report/__tests__/report.test.ts new file mode 100644 index 0000000000..9d765542f5 --- /dev/null +++ b/agentic/pi-ext-usage-report/__tests__/report.test.ts @@ -0,0 +1,86 @@ +import { type AssistantUsageMessage, isAssistantMessage, toUsageReport } from '../src'; + +const usage = { + input: 100, + output: 40, + cacheRead: 900, + cacheWrite: 10, + totalTokens: 1050, + cost: { input: 0.1, output: 0.2, cacheRead: 0.01, cacheWrite: 0.02, total: 0.33 } +}; + +const message: AssistantUsageMessage = { + role: 'assistant', + provider: 'anthropic', + model: 'claude-sonnet-4', + responseId: 'resp-1', + stopReason: 'stop', + usage +}; + +describe('isAssistantMessage', () => { + it('accepts only assistant messages', () => { + expect(isAssistantMessage(message)).toBe(true); + expect(isAssistantMessage({ role: 'user', content: 'hi' })).toBe(false); + expect(isAssistantMessage(null)).toBe(false); + expect(isAssistantMessage('assistant')).toBe(false); + }); +}); + +describe('toUsageReport', () => { + it('counts cached prompt tokens as input so a long session is not under-reported', () => { + expect(toUsageReport(message)).toEqual({ + model: 'claude-sonnet-4', + provider: 'anthropic', + service: 'chat', + operation: 'pi/chat', + input_tokens: 1010, + output_tokens: 40, + total_tokens: 1050, + latency_ms: 0, + status: 'ok', + raw_usage: usage + }); + }); + + it('keeps the full pi usage — cache splits and cost — as raw_usage', () => { + expect(toUsageReport(message)?.raw_usage).toBe(usage); + }); + + it('prefers the model the provider actually answered with', () => { + expect(toUsageReport({ ...message, responseModel: 'claude-sonnet-4-20250514' })?.model).toBe( + 'claude-sonnet-4-20250514' + ); + }); + + it('derives a total when pi reports none', () => { + const report = toUsageReport({ ...message, usage: { input: 5, output: 7, cacheRead: 3 } }); + expect(report).toMatchObject({ input_tokens: 8, output_tokens: 7, total_tokens: 15 }); + }); + + it('marks failed turns and carries the provider error', () => { + const report = toUsageReport({ ...message, stopReason: 'error', errorMessage: 'overloaded' }); + expect(report).toMatchObject({ status: 'error', error_type: 'overloaded' }); + }); + + it('falls back to a generic error type when pi gives no message', () => { + expect(toUsageReport({ ...message, stopReason: 'error' })?.error_type).toBe('error'); + }); + + it('reports nothing when the turn never reached the provider', () => { + expect(toUsageReport({ role: 'assistant', stopReason: 'aborted' })).toBeUndefined(); + }); + + it('tolerates missing provider/model and non-finite counters', () => { + const report = toUsageReport({ role: 'assistant', usage: { input: Number.NaN, output: 3 } }); + expect(report).toMatchObject({ model: 'unknown', provider: 'unknown', input_tokens: 0, output_tokens: 3 }); + }); + + it('rounds and clamps host-measured latency, and honours a custom operation', () => { + expect(toUsageReport(message, { latencyMs: 1234.6, operation: 'pi/code-task' })).toMatchObject({ + latency_ms: 1235, + operation: 'pi/code-task' + }); + expect(toUsageReport(message, { latencyMs: -5 })?.latency_ms).toBe(0); + }); +}); diff --git a/agentic/pi-ext-usage-report/__tests__/reporter.test.ts b/agentic/pi-ext-usage-report/__tests__/reporter.test.ts new file mode 100644 index 0000000000..888d5f6ba9 --- /dev/null +++ b/agentic/pi-ext-usage-report/__tests__/reporter.test.ts @@ -0,0 +1,123 @@ +import { httpUsageSink, type UsageReport, UsageReporter } from '../src'; + +const report: UsageReport = { + model: 'gpt-4o', + provider: 'openai', + service: 'chat', + operation: 'pi/chat', + input_tokens: 10, + output_tokens: 5, + total_tokens: 15, + latency_ms: 0, + status: 'ok' +}; + +describe('httpUsageSink', () => { + const identity = { databaseId: 'db-1', entityId: 'ent-1', actorId: 'act-1', runToken: 'tok-1' }; + + it('posts to /v1/usage on the gateway root with identity headers', async () => { + const fetchMock = jest.fn().mockResolvedValue({ ok: true, status: 202 }); + const sink = httpUsageSink({ gatewayUrl: 'https://gw.example.com/', identity, fetch: fetchMock as never }); + + await sink(report); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('https://gw.example.com/v1/usage'); + expect(init.method).toBe('POST'); + expect(init.headers).toEqual({ + 'Content-Type': 'application/json', + 'X-Database-Id': 'db-1', + 'X-Entity-Id': 'ent-1', + 'X-Actor-Id': 'act-1', + Authorization: 'Bearer tok-1' + }); + expect(JSON.parse(init.body)).toEqual(report); + }); + + it('rejects a gateway URL that already includes /v1', () => { + expect(() => + httpUsageSink({ gatewayUrl: 'https://gw.example.com/v1', identity, fetch: jest.fn() as never }) + ).toThrow(/drop the \/v1/); + }); + + it('requires a databaseId', () => { + expect(() => + httpUsageSink({ gatewayUrl: 'https://gw.example.com', identity: { databaseId: '' }, fetch: jest.fn() as never }) + ).toThrow(/databaseId is required/); + }); + + it('throws with the gateway status and body when the report is rejected', async () => { + const fetchMock = jest.fn().mockResolvedValue({ + ok: false, + status: 400, + text: () => Promise.resolve('model is required') + }); + const sink = httpUsageSink({ gatewayUrl: 'https://gw.example.com', identity, fetch: fetchMock as never }); + + await expect(sink(report)).rejects.toThrow('usage report: gateway rejected the report (400) model is required'); + }); +}); + +describe('UsageReporter', () => { + it('delivers queued reports in order without blocking the caller', async () => { + const seen: string[] = []; + const reporter = new UsageReporter({ + sink: async (item) => { + seen.push(item.model); + } + }); + + reporter.enqueue({ ...report, model: 'a' }); + reporter.enqueue({ ...report, model: 'b' }); + expect(seen).toEqual([]); + + await reporter.flush(); + expect(seen).toEqual(['a', 'b']); + expect(reporter.delivered).toBe(2); + }); + + it('keeps the first failure and rethrows it from flush', async () => { + const reporter = new UsageReporter({ + sink: async (item) => { + throw new Error(`down: ${item.model}`); + } + }); + + reporter.enqueue({ ...report, model: 'a' }); + reporter.enqueue({ ...report, model: 'b' }); + + await expect(reporter.flush()).rejects.toThrow('down: a'); + expect(reporter.delivered).toBe(0); + }); + + it('keeps delivering after a failure so one bad report does not drop the rest', async () => { + const reporter = new UsageReporter({ + sink: async (item) => { + if (item.model === 'a') throw new Error('down'); + } + }); + + reporter.enqueue({ ...report, model: 'a' }); + reporter.enqueue({ ...report, model: 'b' }); + + await expect(reporter.flush()).rejects.toThrow('down'); + expect(reporter.delivered).toBe(1); + }); + + it('does not rethrow the same failure twice', async () => { + const reporter = new UsageReporter({ sink: async () => Promise.reject(new Error('down')) }); + reporter.enqueue(report); + + await expect(reporter.flush()).rejects.toThrow('down'); + await expect(reporter.flush()).resolves.toBeUndefined(); + }); + + it('routes failures to onError instead when the host wants them non-fatal', async () => { + const onError = jest.fn(); + const reporter = new UsageReporter({ sink: async () => Promise.reject(new Error('down')), onError }); + + reporter.enqueue(report); + await expect(reporter.flush()).resolves.toBeUndefined(); + expect(onError).toHaveBeenCalledWith(expect.any(Error), report); + }); +}); diff --git a/agentic/pi-ext-usage-report/jest.config.js b/agentic/pi-ext-usage-report/jest.config.js new file mode 100644 index 0000000000..8a26efd6d0 --- /dev/null +++ b/agentic/pi-ext-usage-report/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/pi-ext-usage-report/package.json b/agentic/pi-ext-usage-report/package.json new file mode 100644 index 0000000000..f54ea73b85 --- /dev/null +++ b/agentic/pi-ext-usage-report/package.json @@ -0,0 +1,48 @@ +{ + "name": "@agentic-kit/pi-ext-usage-report", + "version": "0.1.0", + "author": "Constructive ", + "description": "pi extension that reports each assistant message’s token usage and cost to the Constructive gateway’s /v1/usage endpoint", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "dependencies": { + "@agentic-kit/pi-ext-metered-model": "workspace:^" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": ">=0.79.0" + }, + "devDependencies": { + "@earendil-works/pi-coding-agent": "0.79.6" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "usage", + "metering", + "constructive" + ] +} diff --git a/agentic/pi-ext-usage-report/src/extension.ts b/agentic/pi-ext-usage-report/src/extension.ts new file mode 100644 index 0000000000..eff9a50841 --- /dev/null +++ b/agentic/pi-ext-usage-report/src/extension.ts @@ -0,0 +1,81 @@ +/** + * The pi extension: report each assistant message's usage to the gateway. + * + * This is the local / own-provider-key lane. The agent's own numbers are the only + * source, so a report is *self-reported* — good for reconciliation and for usage + * visibility on a developer's own key, not tamper-proof billing. When the run + * must be billed authoritatively, route the model calls through the gateway with + * `@agentic-kit/pi-ext-metered-model` instead; the gateway then meters itself and + * this extension is unnecessary. + */ + +import type { MeteredIdentity } from '@agentic-kit/pi-ext-metered-model'; +import type { ExtensionAPI, ExtensionFactory } from '@earendil-works/pi-coding-agent'; + +import { isAssistantMessage, toUsageReport, type UsageReport } from './report'; +import { httpUsageSink, UsageReporter } from './reporter'; + +export interface UsageReportExtensionOptions { + identity: MeteredIdentity; + /** Gateway root. Required unless a custom `sink` is supplied. */ + gatewayUrl?: string; + /** Delivery override — e.g. an in-process sink, or a queue. */ + sink?: (report: UsageReport) => Promise; + fetch?: typeof globalThis.fetch; + /** `operation` column value; defaults to `pi/chat`. */ + operation?: string; + /** Report failures here instead of having `flush()` rethrow them. */ + onError?: (error: unknown, report: UsageReport) => void; +} + +export interface UsageReportExtension { + extension: ExtensionFactory; + /** Drain the queue and rethrow the first delivery failure. */ + flush(): Promise; + reporter: UsageReporter; +} + +export function createUsageReportExtension(options: UsageReportExtensionOptions): UsageReportExtension { + const sink = + options.sink ?? + httpUsageSink({ + gatewayUrl: requireGatewayUrl(options), + identity: options.identity, + ...(options.fetch ? { fetch: options.fetch } : {}) + }); + + const reporter = new UsageReporter({ sink, ...(options.onError ? { onError: options.onError } : {}) }); + + // pi can emit `message_end` more than once for the same response (a rewritten + // message, a replayed entry on resume), and each emission would otherwise bill + // again. `responseId` is the provider's identity for the response, so it is + // what dedupes; messages without one are reported as-is. + const seen = new Set(); + + const extension: ExtensionFactory = (pi: ExtensionAPI) => { + pi.on('message_end', (event) => { + const message = event.message; + if (!isAssistantMessage(message)) return; + + const responseId = message.responseId; + if (typeof responseId === 'string' && responseId.length > 0) { + if (seen.has(responseId)) return; + seen.add(responseId); + } + + const report = toUsageReport(message, options.operation === undefined ? {} : { operation: options.operation }); + if (report) reporter.enqueue(report); + }); + + pi.on('session_shutdown', async () => { + await reporter.flush(); + }); + }; + + return { extension, flush: () => reporter.flush(), reporter }; +} + +function requireGatewayUrl(options: UsageReportExtensionOptions): string { + if (!options.gatewayUrl) throw new Error('usage report: gatewayUrl is required unless a custom sink is provided'); + return options.gatewayUrl; +} diff --git a/agentic/pi-ext-usage-report/src/index.ts b/agentic/pi-ext-usage-report/src/index.ts new file mode 100644 index 0000000000..0e7b7d6287 --- /dev/null +++ b/agentic/pi-ext-usage-report/src/index.ts @@ -0,0 +1,25 @@ +/** + * `@agentic-kit/pi-ext-usage-report` — report a pi session's token usage and cost + * to the Constructive gateway's `/v1/usage` endpoint, so runs on the host's own + * provider keys still show up in `inference_log`. + */ + +export { + createUsageReportExtension, + type UsageReportExtension, + type UsageReportExtensionOptions +} from './extension'; +export { + type AssistantUsageMessage, + isAssistantMessage, + toUsageReport, + type ToUsageReportOptions, + type UsageReport +} from './report'; +export { + httpUsageSink, + type HttpUsageSinkOptions, + UsageReporter, + type UsageReporterOptions, + type UsageSink +} from './reporter'; diff --git a/agentic/pi-ext-usage-report/src/report.ts b/agentic/pi-ext-usage-report/src/report.ts new file mode 100644 index 0000000000..374a1323fb --- /dev/null +++ b/agentic/pi-ext-usage-report/src/report.ts @@ -0,0 +1,95 @@ +/** + * Turn a pi assistant message into a `POST /v1/usage` body. + * + * pi reports richer usage than the gateway's own proxy path sees — cache reads, + * cache writes, and its own cost breakdown — so the whole `usage` object is sent + * as `raw_usage` while the four scalar columns carry what billing aggregates. + */ + +/** The slice of pi's `AssistantMessage` this package reads. */ +export interface AssistantUsageMessage { + role: string; + provider?: string; + model?: string; + responseModel?: string; + responseId?: string; + stopReason?: string; + errorMessage?: string; + usage?: { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + totalTokens?: number; + cost?: Record; + }; +} + +/** The gateway's `/v1/usage` payload (snake_case, as the endpoint reads it). */ +export interface UsageReport { + model: string; + provider: string; + service: 'chat'; + operation: string; + input_tokens: number; + output_tokens: number; + total_tokens: number; + latency_ms: number; + status: 'ok' | 'error'; + error_type?: string; + raw_usage?: unknown; +} + +export interface ToUsageReportOptions { + /** Free-form label for the row; defaults to `pi/chat`. */ + operation?: string; + /** Turn latency, when the host measured it. */ + latencyMs?: number; +} + +export function isAssistantMessage(message: unknown): message is AssistantUsageMessage { + return typeof message === 'object' && message !== null && (message as { role?: unknown }).role === 'assistant'; +} + +/** + * `undefined` when the message carries no usage at all (aborted before the + * provider answered), which is a row worth nothing rather than a row of zeros. + */ +export function toUsageReport( + message: AssistantUsageMessage, + options: ToUsageReportOptions = {} +): UsageReport | undefined { + const usage = message.usage; + if (!usage) return undefined; + + const input = num(usage.input); + const cacheRead = num(usage.cacheRead); + const cacheWrite = num(usage.cacheWrite); + const output = num(usage.output); + + // Every prompt token the provider processed, cached or not: `input` alone + // excludes cache hits, which would under-report a long agent session + // dramatically. The split survives in `raw_usage`. + const inputTokens = input + cacheRead + cacheWrite; + const totalTokens = usage.totalTokens === undefined ? inputTokens + output : num(usage.totalTokens); + + const failed = message.stopReason === 'error'; + const report: UsageReport = { + model: message.responseModel ?? message.model ?? 'unknown', + provider: message.provider ?? 'unknown', + service: 'chat', + operation: options.operation ?? 'pi/chat', + input_tokens: inputTokens, + output_tokens: output, + total_tokens: totalTokens, + latency_ms: options.latencyMs === undefined ? 0 : Math.max(0, Math.round(options.latencyMs)), + status: failed ? 'error' : 'ok', + raw_usage: usage + }; + if (failed) report.error_type = message.errorMessage ?? 'error'; + return report; +} + +function num(value: number | undefined): number { + return typeof value === 'number' && Number.isFinite(value) ? value : 0; +} diff --git a/agentic/pi-ext-usage-report/src/reporter.ts b/agentic/pi-ext-usage-report/src/reporter.ts new file mode 100644 index 0000000000..dca6ef7ba6 --- /dev/null +++ b/agentic/pi-ext-usage-report/src/reporter.ts @@ -0,0 +1,90 @@ +/** + * Delivery of usage reports: a serialized queue plus the default HTTP sink. + * + * Reports are queued rather than awaited inside pi's event handler — a network + * round trip per assistant message would sit directly in the agent's turn + * latency. Failures are therefore not lost: the first one is kept and rethrown + * from `flush()`, so a host that flushes at shutdown still fails loudly. + */ + +import { buildIdentityHeaders, type MeteredIdentity, normalizeGatewayUrl } from '@agentic-kit/pi-ext-metered-model'; + +import type { UsageReport } from './report'; + +export type UsageSink = (report: UsageReport) => Promise; + +export interface HttpUsageSinkOptions { + /** Gateway root; `/v1/usage` is appended. */ + gatewayUrl: string; + identity: MeteredIdentity; + /** Injectable for tests and for hosts with a custom agent/proxy. */ + fetch?: typeof globalThis.fetch; +} + +export function httpUsageSink(options: HttpUsageSinkOptions): UsageSink { + const url = `${normalizeGatewayUrl(options.gatewayUrl)}/v1/usage`; + const headers = { 'Content-Type': 'application/json', ...buildIdentityHeaders(options.identity) }; + const doFetch = options.fetch ?? globalThis.fetch; + if (!doFetch) throw new Error('usage report: no fetch implementation available; pass options.fetch'); + + return async (report) => { + const response = await doFetch(url, { method: 'POST', headers, body: JSON.stringify(report) }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`usage report: gateway rejected the report (${response.status}) ${body}`.trim()); + } + }; +} + +export interface UsageReporterOptions { + sink: UsageSink; + /** Called instead of retaining the error for `flush()` to rethrow. */ + onError?: (error: unknown, report: UsageReport) => void; +} + +export class UsageReporter { + private readonly sink: UsageSink; + private readonly onError: UsageReporterOptions['onError']; + + private tail: Promise = Promise.resolve(); + private failure: unknown; + private sent = 0; + + constructor(options: UsageReporterOptions) { + this.sink = options.sink; + this.onError = options.onError; + } + + /** Reports delivered successfully so far. */ + get delivered(): number { + return this.sent; + } + + /** Queue a report. Never rejects; the failure surfaces from `flush()`. */ + enqueue(report: UsageReport): void { + this.tail = this.tail.then(async (): Promise => { + try { + await this.sink(report); + this.sent += 1; + } catch (error) { + if (this.onError) { + this.onError(error, report); + return; + } + // Keep the first failure: it is the one with the original cause, and a + // later cascade usually says less about what broke. + if (this.failure === undefined) this.failure = error; + } + }); + } + + /** Wait for the queue to drain, then rethrow the first retained failure. */ + async flush(): Promise { + await this.tail; + const failure = this.failure; + if (failure !== undefined) { + this.failure = undefined; + throw failure; + } + } +} diff --git a/agentic/pi-ext-usage-report/tsconfig.esm.json b/agentic/pi-ext-usage-report/tsconfig.esm.json new file mode 100644 index 0000000000..624ab17cfa --- /dev/null +++ b/agentic/pi-ext-usage-report/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/pi-ext-usage-report/tsconfig.json b/agentic/pi-ext-usage-report/tsconfig.json new file mode 100644 index 0000000000..df063b5ee3 --- /dev/null +++ b/agentic/pi-ext-usage-report/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 567d265dd7..d1e611c325 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -262,6 +262,17 @@ importers: version: 0.79.6(ws@8.20.1)(zod@4.4.3) publishDirectory: dist + agentic/pi-ext-usage-report: + dependencies: + '@agentic-kit/pi-ext-metered-model': + specifier: workspace:^ + version: link:../pi-ext-metered-model/dist + devDependencies: + '@earendil-works/pi-coding-agent': + specifier: 0.79.6 + version: 0.79.6(ws@8.20.1)(zod@4.4.3) + publishDirectory: dist + agentic/pi-ext-run-log: dependencies: '@agentic-kit/run-log': From 32e5631ae0218a88d3c845ec6ee0b984f8538a73 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 14 Aug 2026 04:24:53 +0000 Subject: [PATCH 5/5] feat(pi-ext-gate): gate pi tool calls through policy and remote approvals --- .github/workflows/run-tests.yaml | 2 +- agentic/pi-ext-gate/README.md | 71 +++++++ .../pi-ext-gate/__tests__/approval.test.ts | 96 +++++++++ .../pi-ext-gate/__tests__/extension.test.ts | 182 ++++++++++++++++++ agentic/pi-ext-gate/__tests__/policy.test.ts | 66 +++++++ agentic/pi-ext-gate/jest.config.js | 21 ++ agentic/pi-ext-gate/package.json | 45 +++++ agentic/pi-ext-gate/src/approval.ts | 80 ++++++++ agentic/pi-ext-gate/src/extension.ts | 115 +++++++++++ agentic/pi-ext-gate/src/index.ts | 29 +++ agentic/pi-ext-gate/src/policy.ts | 72 +++++++ agentic/pi-ext-gate/tsconfig.esm.json | 7 + agentic/pi-ext-gate/tsconfig.json | 8 + pnpm-lock.yaml | 7 + 14 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 agentic/pi-ext-gate/README.md create mode 100644 agentic/pi-ext-gate/__tests__/approval.test.ts create mode 100644 agentic/pi-ext-gate/__tests__/extension.test.ts create mode 100644 agentic/pi-ext-gate/__tests__/policy.test.ts create mode 100644 agentic/pi-ext-gate/jest.config.js create mode 100644 agentic/pi-ext-gate/package.json create mode 100644 agentic/pi-ext-gate/src/approval.ts create mode 100644 agentic/pi-ext-gate/src/extension.ts create mode 100644 agentic/pi-ext-gate/src/index.ts create mode 100644 agentic/pi-ext-gate/src/policy.ts create mode 100644 agentic/pi-ext-gate/tsconfig.esm.json create mode 100644 agentic/pi-ext-gate/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index a02731934c..ba3a5193f7 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -115,7 +115,7 @@ jobs: - batch: graphile-unit packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin' - batch: agentic - packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log agentic/pi-ext-metered-model agentic/pi-ext-usage-report' + packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/chat agentic/cli agentic/pi agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/pi-ext-run-log agentic/pi-ext-metered-model agentic/pi-ext-usage-report agentic/pi-ext-gate' - batch: pgpm-unit packages: 'pgpm/types pgpm/naming-spec pgpm/diff pgpm/import pgpm/slice pgpm/transform' - batch: pglite diff --git a/agentic/pi-ext-gate/README.md b/agentic/pi-ext-gate/README.md new file mode 100644 index 0000000000..cded4d87e5 --- /dev/null +++ b/agentic/pi-ext-gate/README.md @@ -0,0 +1,71 @@ +

+ +

+ +# @agentic-kit/pi-ext-gate + +A pi extension that gates tool calls through a **declared policy**, and defers anything the policy will not decide alone to an **approval channel** that can reach a human on another machine. + +This is what makes an unattended cloud run safe to start: the run's permissions are data, decided before it begins, and the escape hatch for everything else is an approval that travels over the platform's existing rows/API rather than a socket back into the cluster. + +## Policy + +A policy is an ordered rule list evaluated with no I/O — so the same value can be asserted in a test, stored on a run row, and rendered in a UI. + +```ts +import { gatePolicy } from '@agentic-kit/pi-ext-gate'; + +const policy = gatePolicy({ + rules: [ + { tool: 'bash', decision: 'ask', reason: 'destructive command', match: ({ input }) => /\brm\b/.test(String(input.command)) }, + { tool: 'bash', decision: 'allow' }, + { tool: 'write', decision: 'ask', reason: 'writes to the workspace' }, + { tool: '*', decision: 'allow' } + ], + defaultDecision: 'deny' +}); +``` + +First match wins, so order the list most specific first. With no `defaultDecision` the default is `ask`, not `allow`: a tool nobody declared is not implicitly trusted, and denying it outright would strand the run. + +## Approvals + +```ts +import { createGateExtension, pollingApprovalChannel } from '@agentic-kit/pi-ext-gate'; + +const gate = createGateExtension({ + runId, + policy, + approvals: pollingApprovalChannel({ + submit: (request) => api.createApprovalRequest(request), // a row + poll: (request) => api.readApprovalDecision(request), // another row + intervalMs: 1000, + timeoutMs: 15 * 60_000 + }), + onDecision: (record) => runLog.append(record) +}); + +// hand `gate.extension` to pi +``` + +pi's `tool_call` handler can await, so an `ask` verdict simply suspends that one tool until the channel resolves — the rest of the session is untouched. `gate.pending` exposes the requests currently waiting. + +## Behavior + +- **Deny → `{ block: true, reason }`.** The reason is the model's *only* explanation for the refusal, so write policy reasons for the model to read. +- **Timeouts deny by default.** An unattended run must not take an unapproved action just because nobody was watching; `onTimeout: 'allow'` opts out. +- **A channel failure is not an approval.** `submit`/`poll` errors propagate and the tool call fails, rather than degrading into a silent allow. +- **Misconfiguration fails at construction:** a policy that can produce `ask` with no `approvals` channel throws immediately instead of on the first sensitive tool call. +- **Every settled decision is emitted** to `onDecision` — the policy verdict, the final decision, the deciding actor, the timestamp — which is what an audit trail or a run log stores. + +`staticApprovalChannel({ decision: 'allow' })` is the explicit auto-approve for tests and deliberately unattended runs; it exists so that "approve everything" has to be written down. + +## Testing + +`pnpm test` — the policy is pure, the channel takes an injected clock/sleep, and the extension is driven through a fake `ExtensionAPI`, so nothing here needs a network or a model. + +## Related + +- `@agentic-kit/run-log` — append-only run log (`onDecision` records belong there) +- `@agentic-kit/pi-ext-run-log` — mirrors pi session entries into it +- `@agentic-kit/pi-ext-metered-model` / `@agentic-kit/pi-ext-usage-report` — the two metering lanes diff --git a/agentic/pi-ext-gate/__tests__/approval.test.ts b/agentic/pi-ext-gate/__tests__/approval.test.ts new file mode 100644 index 0000000000..d211d87d34 --- /dev/null +++ b/agentic/pi-ext-gate/__tests__/approval.test.ts @@ -0,0 +1,96 @@ +import { type ApprovalOutcome, type ApprovalRequest, pollingApprovalChannel, staticApprovalChannel } from '../src'; + +const request: ApprovalRequest = { + runId: 'run-1', + toolCallId: 'call-1', + toolName: 'bash', + input: { command: 'rm -rf build' }, + reason: 'destructive command', + requestedAt: '2026-01-01T00:00:00.000Z' +}; + +describe('pollingApprovalChannel', () => { + it('submits once, then polls until a decision exists', async () => { + const submit = jest.fn().mockResolvedValue(undefined); + const outcome: ApprovalOutcome = { decision: 'allow', actorId: 'user-1' }; + const poll = jest + .fn, [ApprovalRequest]>() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(outcome); + + const channel = pollingApprovalChannel({ submit, poll, sleep: () => Promise.resolve() }); + + await expect(channel.request(request)).resolves.toEqual(outcome); + expect(submit).toHaveBeenCalledTimes(1); + expect(submit).toHaveBeenCalledWith(request); + expect(poll).toHaveBeenCalledTimes(3); + }); + + it('waits between polls at the configured interval', async () => { + const sleep = jest.fn().mockResolvedValue(undefined); + const channel = pollingApprovalChannel({ + submit: () => Promise.resolve(), + poll: jest.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce({ decision: 'deny' }), + intervalMs: 250, + sleep + }); + + await channel.request(request); + expect(sleep).toHaveBeenCalledWith(250); + }); + + it('denies on timeout, because an unattended run must not act unapproved', async () => { + let clock = 0; + const channel = pollingApprovalChannel({ + submit: () => Promise.resolve(), + poll: () => Promise.resolve(undefined), + timeoutMs: 500, + intervalMs: 100, + sleep: () => { + clock += 100; + return Promise.resolve(); + }, + now: () => clock + }); + + await expect(channel.request(request)).resolves.toEqual({ + decision: 'deny', + reason: 'gate: no decision within 500ms' + }); + }); + + it('can be configured to allow on timeout instead', async () => { + let clock = 0; + const channel = pollingApprovalChannel({ + submit: () => Promise.resolve(), + poll: () => Promise.resolve(undefined), + timeoutMs: 100, + onTimeout: 'allow', + sleep: () => { + clock += 100; + return Promise.resolve(); + }, + now: () => clock + }); + + await expect(channel.request(request)).resolves.toMatchObject({ decision: 'allow' }); + }); + + it('propagates a submit failure instead of silently waiting forever', async () => { + const channel = pollingApprovalChannel({ + submit: () => Promise.reject(new Error('api down')), + poll: jest.fn(), + sleep: () => Promise.resolve() + }); + + await expect(channel.request(request)).rejects.toThrow('api down'); + }); +}); + +describe('staticApprovalChannel', () => { + it('decides every request the same way', async () => { + const channel = staticApprovalChannel({ decision: 'allow', reason: 'auto-approved run' }); + await expect(channel.request(request)).resolves.toEqual({ decision: 'allow', reason: 'auto-approved run' }); + }); +}); diff --git a/agentic/pi-ext-gate/__tests__/extension.test.ts b/agentic/pi-ext-gate/__tests__/extension.test.ts new file mode 100644 index 0000000000..fd5e38dda3 --- /dev/null +++ b/agentic/pi-ext-gate/__tests__/extension.test.ts @@ -0,0 +1,182 @@ +import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; + +import { + type ApprovalOutcome, + type ApprovalRequest, + createGateExtension, + type GateDecisionRecord, + staticApprovalChannel +} from '../src'; + +type Handler = (event: any, ctx: any) => unknown; + +const fakePi = () => { + const handlers = new Map(); + const pi = { + on: (event: string, handler: Handler) => { + handlers.set(event, handler); + } + } as unknown as ExtensionAPI; + return { + pi, + toolCall: (toolName: string, input: Record, toolCallId = 'call-1') => + handlers.get('tool_call')?.({ type: 'tool_call', toolCallId, toolName, input }, {}) as Promise< + { block?: boolean; reason?: string } | undefined + > + }; +}; + +describe('createGateExtension', () => { + it('lets an allowed tool run', async () => { + const host = fakePi(); + createGateExtension({ + runId: 'run-1', + policy: { rules: [{ tool: 'read', decision: 'allow' }], defaultDecision: 'deny' } + }).extension(host.pi); + + await expect(host.toolCall('read', { path: 'a.ts' })).resolves.toEqual({}); + }); + + it('blocks a denied tool with the policy reason, which is all the model sees', async () => { + const host = fakePi(); + createGateExtension({ + runId: 'run-1', + policy: { rules: [{ tool: 'bash', decision: 'deny', reason: 'no shell in this run' }], defaultDecision: 'allow' } + }).extension(host.pi); + + await expect(host.toolCall('bash', { command: 'ls' })).resolves.toEqual({ + block: true, + reason: 'no shell in this run' + }); + }); + + it('blocks with a generic reason when the policy gave none', async () => { + const host = fakePi(); + createGateExtension({ runId: 'run-1', policy: { defaultDecision: 'deny' } }).extension(host.pi); + + await expect(host.toolCall('write', { path: 'a.ts' })).resolves.toEqual({ + block: true, + reason: 'gate: write is not permitted in this run' + }); + }); + + it('suspends an asked tool call until the channel decides', async () => { + let resolveOutcome: (outcome: ApprovalOutcome) => void = () => undefined; + const seen: ApprovalRequest[] = []; + const gate = createGateExtension({ + runId: 'run-1', + policy: { defaultDecision: 'ask', defaultReason: 'needs review' }, + approvals: { + request: (req) => { + seen.push(req); + return new Promise((resolve) => { + resolveOutcome = resolve; + }); + } + } + }); + const host = fakePi(); + gate.extension(host.pi); + + const call = host.toolCall('bash', { command: 'rm -rf build' }); + await Promise.resolve(); + + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ + runId: 'run-1', + toolCallId: 'call-1', + toolName: 'bash', + input: { command: 'rm -rf build' }, + reason: 'needs review' + }); + expect(gate.pending.get('call-1')).toBeDefined(); + + resolveOutcome({ decision: 'allow', actorId: 'user-1' }); + await expect(call).resolves.toEqual({}); + expect(gate.pending.size).toBe(0); + }); + + it('blocks when the human denies, carrying their reason back to the model', async () => { + const host = fakePi(); + createGateExtension({ + runId: 'run-1', + policy: { defaultDecision: 'ask' }, + approvals: staticApprovalChannel({ decision: 'deny', reason: 'not on production data' }) + }).extension(host.pi); + + await expect(host.toolCall('bash', { command: 'psql' })).resolves.toEqual({ + block: true, + reason: 'not on production data' + }); + }); + + it('stops tracking a request even when the channel fails', async () => { + const gate = createGateExtension({ + runId: 'run-1', + policy: { defaultDecision: 'ask' }, + approvals: { request: () => Promise.reject(new Error('api down')) } + }); + const host = fakePi(); + gate.extension(host.pi); + + await expect(host.toolCall('bash', {})).rejects.toThrow('api down'); + expect(gate.pending.size).toBe(0); + }); + + it('records every settled decision for the audit trail', async () => { + const records: GateDecisionRecord[] = []; + const host = fakePi(); + createGateExtension({ + runId: 'run-1', + policy: { rules: [{ tool: 'read', decision: 'allow' }], defaultDecision: 'ask' }, + approvals: staticApprovalChannel({ decision: 'deny', reason: 'rejected', actorId: 'user-1' }), + onDecision: (record) => records.push(record), + now: () => new Date('2026-01-01T00:00:00.000Z') + }).extension(host.pi); + + await host.toolCall('read', { path: 'a.ts' }, 'call-a'); + await host.toolCall('bash', { command: 'ls' }, 'call-b'); + + expect(records).toEqual([ + { + runId: 'run-1', + toolCallId: 'call-a', + toolName: 'read', + input: { path: 'a.ts' }, + verdict: { decision: 'allow', rule: { tool: 'read', decision: 'allow' } }, + decision: 'allow', + decidedAt: '2026-01-01T00:00:00.000Z' + }, + { + runId: 'run-1', + toolCallId: 'call-b', + toolName: 'bash', + input: { command: 'ls' }, + verdict: { decision: 'ask' }, + decision: 'deny', + reason: 'rejected', + actorId: 'user-1', + decidedAt: '2026-01-01T00:00:00.000Z' + } + ]); + }); + + it('refuses to start a policy that can ask with nowhere to ask', () => { + expect(() => createGateExtension({ runId: 'run-1', policy: {} })).toThrow(/approvals channel is required/); + expect(() => + createGateExtension({ + runId: 'run-1', + policy: { rules: [{ tool: 'bash', decision: 'ask' }], defaultDecision: 'allow' } + }) + ).toThrow(/approvals channel is required/); + }); + + it('needs no channel when the policy only allows and denies', () => { + expect(() => + createGateExtension({ + runId: 'run-1', + policy: { rules: [{ tool: 'bash', decision: 'deny' }], defaultDecision: 'allow' } + }) + ).not.toThrow(); + }); +}); diff --git a/agentic/pi-ext-gate/__tests__/policy.test.ts b/agentic/pi-ext-gate/__tests__/policy.test.ts new file mode 100644 index 0000000000..194616d546 --- /dev/null +++ b/agentic/pi-ext-gate/__tests__/policy.test.ts @@ -0,0 +1,66 @@ +import { gatePolicy } from '../src'; + +describe('gatePolicy', () => { + it('asks by default so an undeclared tool is never implicitly trusted', () => { + expect(gatePolicy().evaluate({ toolName: 'bash', input: {} })).toEqual({ decision: 'ask' }); + }); + + it('lets the first matching rule win', () => { + const policy = gatePolicy({ + rules: [ + { tool: 'bash', decision: 'deny', reason: 'no shell in this run' }, + { tool: '*', decision: 'allow' } + ] + }); + + expect(policy.evaluate({ toolName: 'bash', input: {} })).toMatchObject({ + decision: 'deny', + reason: 'no shell in this run' + }); + expect(policy.evaluate({ toolName: 'read', input: {} })).toMatchObject({ decision: 'allow' }); + }); + + it('narrows a rule by tool arguments', () => { + const policy = gatePolicy({ + rules: [ + { + tool: 'bash', + decision: 'ask', + reason: 'destructive command', + match: ({ input }) => String(input.command ?? '').includes('rm ') + }, + { tool: 'bash', decision: 'allow' } + ], + defaultDecision: 'deny' + }); + + expect(policy.evaluate({ toolName: 'bash', input: { command: 'rm -rf build' } })).toMatchObject({ + decision: 'ask', + reason: 'destructive command' + }); + expect(policy.evaluate({ toolName: 'bash', input: { command: 'ls' } })).toMatchObject({ decision: 'allow' }); + }); + + it('falls through to the configured default', () => { + const policy = gatePolicy({ + rules: [{ tool: 'read', decision: 'allow' }], + defaultDecision: 'deny', + defaultReason: 'read-only run' + }); + + expect(policy.evaluate({ toolName: 'write', input: {} })).toEqual({ + decision: 'deny', + reason: 'read-only run' + }); + }); + + it('reports the rule that decided, so an audit trail can name it', () => { + const rule = { tool: 'write' as const, decision: 'deny' as const }; + const verdict = gatePolicy({ rules: [rule] }).evaluate({ toolName: 'write', input: {} }); + expect(verdict.rule).toBe(rule); + }); + + it('rejects a rule with no tool name', () => { + expect(() => gatePolicy({ rules: [{ tool: '', decision: 'allow' }] })).toThrow(/needs a tool name/); + }); +}); diff --git a/agentic/pi-ext-gate/jest.config.js b/agentic/pi-ext-gate/jest.config.js new file mode 100644 index 0000000000..8a26efd6d0 --- /dev/null +++ b/agentic/pi-ext-gate/jest.config.js @@ -0,0 +1,21 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + babelConfig: false, + tsconfig: 'tsconfig.json', + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: '(/__tests__/.*\\.(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, +}; diff --git a/agentic/pi-ext-gate/package.json b/agentic/pi-ext-gate/package.json new file mode 100644 index 0000000000..a69ddb0064 --- /dev/null +++ b/agentic/pi-ext-gate/package.json @@ -0,0 +1,45 @@ +{ + "name": "@agentic-kit/pi-ext-gate", + "version": "0.1.0", + "author": "Constructive ", + "description": "pi extension that gates tool calls through a declared policy and a remote approval channel", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/constructive", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/constructive" + }, + "bugs": { + "url": "https://github.com/constructive-io/constructive/issues" + }, + "scripts": { + "clean": "makage clean", + "prepack": "npm run build", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "peerDependencies": { + "@earendil-works/pi-coding-agent": ">=0.79.0" + }, + "devDependencies": { + "@earendil-works/pi-coding-agent": "0.79.6" + }, + "keywords": [ + "agentic-kit", + "pi", + "coding-agent", + "approvals", + "policy", + "constructive" + ] +} diff --git a/agentic/pi-ext-gate/src/approval.ts b/agentic/pi-ext-gate/src/approval.ts new file mode 100644 index 0000000000..7e86a76282 --- /dev/null +++ b/agentic/pi-ext-gate/src/approval.ts @@ -0,0 +1,80 @@ +/** + * The asynchronous half of the gate: an `ask` verdict has to reach a human who + * may be on another machine, so approval is an interface rather than a prompt. + * + * `pollingApprovalChannel` is the baseline implementation of the platform's + * rendezvous model — the request is submitted (a row), and the decision is + * polled for (another row). No socket back into the agent's process, which is + * what lets a cloud run be approved from a browser. + */ + +export interface ApprovalRequest { + runId: string; + toolCallId: string; + toolName: string; + input: Record; + /** The policy's reason for asking. */ + reason?: string; + requestedAt: string; +} + +export interface ApprovalOutcome { + decision: 'allow' | 'deny'; + reason?: string; + /** Who decided, when the channel knows. */ + actorId?: string; +} + +export interface ApprovalChannel { + /** Resolves once a decision exists; may take as long as a human takes. */ + request(request: ApprovalRequest): Promise; +} + +export interface PollingApprovalChannelOptions { + /** Publish the request (insert the row, post to the API). */ + submit: (request: ApprovalRequest) => Promise; + /** Read the decision, or `undefined` while it is still outstanding. */ + poll: (request: ApprovalRequest) => Promise; + intervalMs?: number; + /** Give up after this long. Omit to wait indefinitely. */ + timeoutMs?: number; + /** Verdict on timeout. Defaults to `deny`: an unattended run must not take an + * action nobody approved just because nobody was watching. */ + onTimeout?: 'deny' | 'allow'; + /** Injectable clock/sleep for tests. */ + sleep?: (ms: number) => Promise; + now?: () => number; +} + +export function pollingApprovalChannel(options: PollingApprovalChannelOptions): ApprovalChannel { + const intervalMs = options.intervalMs ?? 1000; + const onTimeout = options.onTimeout ?? 'deny'; + const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const now = options.now ?? (() => Date.now()); + + return { + async request(request) { + await options.submit(request); + const startedAt = now(); + + for (;;) { + const outcome = await options.poll(request); + if (outcome) return outcome; + + if (options.timeoutMs !== undefined && now() - startedAt >= options.timeoutMs) { + return { + decision: onTimeout, + reason: `gate: no decision within ${options.timeoutMs}ms` + }; + } + await sleep(intervalMs); + } + } + }; +} + +/** Decide every request the same way — for tests, and for a headless run that is + * explicitly configured to auto-approve. */ +export function staticApprovalChannel(outcome: ApprovalOutcome): ApprovalChannel { + return { request: () => Promise.resolve(outcome) }; +} diff --git a/agentic/pi-ext-gate/src/extension.ts b/agentic/pi-ext-gate/src/extension.ts new file mode 100644 index 0000000000..cc091107e5 --- /dev/null +++ b/agentic/pi-ext-gate/src/extension.ts @@ -0,0 +1,115 @@ +/** + * The pi extension: evaluate the policy on every `tool_call`, and block the call + * when the policy — or the human the policy defers to — says no. + * + * pi's `tool_call` handler can await, so an `ask` verdict simply suspends the + * tool until the approval channel resolves. That is the whole mechanism for + * "approve a cloud agent's `rm -rf` from a browser tab". + */ + +import type { ExtensionAPI, ExtensionFactory } from '@earendil-works/pi-coding-agent'; + +import { type ApprovalChannel, type ApprovalOutcome, type ApprovalRequest } from './approval'; +import { type GateDecision, GatePolicy, type GatePolicyOptions, type GateVerdict } from './policy'; + +export interface GateDecisionRecord { + runId: string; + toolCallId: string; + toolName: string; + input: Record; + /** What the policy said. */ + verdict: GateVerdict; + /** How the request was finally settled. */ + decision: 'allow' | 'deny'; + reason?: string; + actorId?: string; + decidedAt: string; +} + +export interface GateExtensionOptions { + runId: string; + /** A policy, or the options to build one. */ + policy: GatePolicy | GatePolicyOptions; + /** Required only if the policy can produce `ask`. */ + approvals?: ApprovalChannel; + /** Every settled decision, for the run log / audit trail. */ + onDecision?: (record: GateDecisionRecord) => void; + now?: () => Date; +} + +export interface GateExtension { + extension: ExtensionFactory; + policy: GatePolicy; + /** Requests currently waiting on a human. */ + pending: ReadonlyMap; +} + +export function createGateExtension(options: GateExtensionOptions): GateExtension { + const policy = options.policy instanceof GatePolicy ? options.policy : new GatePolicy(options.policy); + const now = options.now ?? (() => new Date()); + + if (canAsk(policy) && !options.approvals) { + throw new Error('gate: the policy can produce "ask", so an approvals channel is required'); + } + + const pending = new Map(); + + const extension: ExtensionFactory = (pi: ExtensionAPI) => { + pi.on('tool_call', async (event) => { + const input = (event.input ?? {}) as Record; + const verdict = policy.evaluate({ toolName: event.toolName, input }); + + const settled: ApprovalOutcome = + verdict.decision === 'ask' + ? await ask(event.toolCallId, event.toolName, input, verdict) + : { decision: verdict.decision, ...(verdict.reason === undefined ? {} : { reason: verdict.reason }) }; + + options.onDecision?.({ + runId: options.runId, + toolCallId: event.toolCallId, + toolName: event.toolName, + input, + verdict, + decision: settled.decision, + ...(settled.reason === undefined ? {} : { reason: settled.reason }), + ...(settled.actorId === undefined ? {} : { actorId: settled.actorId }), + decidedAt: now().toISOString() + }); + + if (settled.decision === 'allow') return {}; + // The reason is the model's only explanation for the refusal, so it is + // worth being specific in the policy. + return { block: true, reason: settled.reason ?? `gate: ${event.toolName} is not permitted in this run` }; + }); + }; + + return { extension, policy, pending }; + + async function ask( + toolCallId: string, + toolName: string, + input: Record, + verdict: GateVerdict + ): Promise { + const request: ApprovalRequest = { + runId: options.runId, + toolCallId, + toolName, + input, + ...(verdict.reason === undefined ? {} : { reason: verdict.reason }), + requestedAt: now().toISOString() + }; + + pending.set(toolCallId, request); + try { + return await options.approvals!.request(request); + } finally { + pending.delete(toolCallId); + } + } +} + +function canAsk(policy: GatePolicy): boolean { + const decisions: GateDecision[] = [policy.defaultDecision, ...policy.rules.map((rule) => rule.decision)]; + return decisions.includes('ask'); +} diff --git a/agentic/pi-ext-gate/src/index.ts b/agentic/pi-ext-gate/src/index.ts new file mode 100644 index 0000000000..44bafba2a7 --- /dev/null +++ b/agentic/pi-ext-gate/src/index.ts @@ -0,0 +1,29 @@ +/** + * `@agentic-kit/pi-ext-gate` — gate a pi session's tool calls through a declared + * policy, deferring anything the policy will not decide alone to an approval + * channel that can reach a human on another machine. + */ + +export { + type ApprovalChannel, + type ApprovalOutcome, + type ApprovalRequest, + pollingApprovalChannel, + type PollingApprovalChannelOptions, + staticApprovalChannel +} from './approval'; +export { + createGateExtension, + type GateDecisionRecord, + type GateExtension, + type GateExtensionOptions +} from './extension'; +export { + type GateDecision, + GatePolicy, + gatePolicy, + type GatePolicyOptions, + type GateRequest, + type GateRule, + type GateVerdict +} from './policy'; diff --git a/agentic/pi-ext-gate/src/policy.ts b/agentic/pi-ext-gate/src/policy.ts new file mode 100644 index 0000000000..e9cfb5e6f9 --- /dev/null +++ b/agentic/pi-ext-gate/src/policy.ts @@ -0,0 +1,72 @@ +/** + * The declared half of the gate: which tool calls run, which are refused, and + * which need a human. A policy is data — an ordered rule list evaluated with no + * I/O — so the same policy can be asserted in a test, stored on a run row, and + * shown in a UI. + */ + +export type GateDecision = 'allow' | 'deny' | 'ask'; + +export interface GateRequest { + toolName: string; + input: Record; +} + +export interface GateRule { + /** Tool name, or `*` for every tool. */ + tool: string; + decision: GateDecision; + /** Narrow the rule to particular arguments (e.g. a bash command pattern). */ + match?: (request: GateRequest) => boolean; + /** Shown to the model when denied, and to the human when asked. */ + reason?: string; +} + +export interface GateVerdict { + decision: GateDecision; + reason?: string; + /** The rule that decided, or `undefined` when the default applied. */ + rule?: GateRule; +} + +export interface GatePolicyOptions { + rules?: readonly GateRule[]; + /** Verdict when no rule matches. Defaults to `ask` — an undeclared tool is + * not implicitly trusted, and denying outright would strand the run. */ + defaultDecision?: GateDecision; + defaultReason?: string; +} + +export class GatePolicy { + readonly rules: readonly GateRule[]; + readonly defaultDecision: GateDecision; + private readonly defaultReason: string | undefined; + + constructor(options: GatePolicyOptions = {}) { + this.rules = options.rules ?? []; + this.defaultDecision = options.defaultDecision ?? 'ask'; + this.defaultReason = options.defaultReason; + + for (const rule of this.rules) { + if (!rule.tool) throw new Error('gate: every rule needs a tool name (or "*")'); + } + } + + /** First matching rule wins, so order the list most specific first. */ + evaluate(request: GateRequest): GateVerdict { + for (const rule of this.rules) { + if (rule.tool !== '*' && rule.tool !== request.toolName) continue; + if (rule.match && !rule.match(request)) continue; + return rule.reason === undefined + ? { decision: rule.decision, rule } + : { decision: rule.decision, reason: rule.reason, rule }; + } + return this.defaultReason === undefined + ? { decision: this.defaultDecision } + : { decision: this.defaultDecision, reason: this.defaultReason }; + } +} + +export function gatePolicy(options: GatePolicyOptions = {}): GatePolicy { + return new GatePolicy(options); +} diff --git a/agentic/pi-ext-gate/tsconfig.esm.json b/agentic/pi-ext-gate/tsconfig.esm.json new file mode 100644 index 0000000000..624ab17cfa --- /dev/null +++ b/agentic/pi-ext-gate/tsconfig.esm.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "outDir": "dist/esm" + } +} diff --git a/agentic/pi-ext-gate/tsconfig.json b/agentic/pi-ext-gate/tsconfig.json new file mode 100644 index 0000000000..df063b5ee3 --- /dev/null +++ b/agentic/pi-ext-gate/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1e611c325..3f353065ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,13 @@ importers: agentic/protocol: publishDirectory: dist + agentic/pi-ext-gate: + devDependencies: + '@earendil-works/pi-coding-agent': + specifier: 0.79.6 + version: 0.79.6(ws@8.20.1)(zod@4.4.3) + publishDirectory: dist + agentic/pi-ext-metered-model: devDependencies: '@earendil-works/pi-coding-agent':