From 0f557cd01c3312a89377ff8ba81cccea8d0b1517 Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Thu, 16 Jul 2026 00:04:06 +0300 Subject: [PATCH] feat(framework): sync the system prompt with the #326 doc (#555) The shipped prompt was the 11-Jul draft. The #326 doc was rewritten on 13-Jul and never synced, so every run since has been driven by a stale prompt. prompts/system_prompt.md is byte-identical to the doc again. Also retarget the eco flags, which the sync breaks silently: they drop sections by exact heading string and the rewrite renames or moves all three, and a missing heading is a no-op. --- .changeset/sync-system-prompt-555.md | 13 +++ packages/framework/prompts/protocols/await.md | 1 + packages/framework/prompts/system_prompt.md | 55 ++++++--- packages/framework/src/prompt-run.test.ts | 10 +- packages/framework/src/system-prompt.test.ts | 108 ++++++++++++------ packages/framework/src/system-prompt.ts | 66 +++++++---- 6 files changed, 174 insertions(+), 79 deletions(-) create mode 100644 .changeset/sync-system-prompt-555.md diff --git a/.changeset/sync-system-prompt-555.md b/.changeset/sync-system-prompt-555.md new file mode 100644 index 00000000..e240b30e --- /dev/null +++ b/.changeset/sync-system-prompt-555.md @@ -0,0 +1,13 @@ +--- +'@gemstack/framework': minor +--- + +Sync the built-in system prompt with the #326 doc (#555). The shipped prompt was the 11-Jul draft: the doc was rewritten on 13-Jul and never synced, so every run since has been driven by a stale prompt. `prompts/system_prompt.md` is now byte-identical to the doc again. + +What the agent gets that it did not before: `## Analyze the user prompt`, which analyzes the prompt up front and records the results in an `ANLYSIS_RESULT.md`; `## Before starting changes` -> `### Session name`, which commits any dirty tree, then creates and checks out a `the-framework/` branch before the first change and calls setSessionName(); and `## After applying changes`, which calls setReadyForMerge() once the session has no work left. `## Unclear scope` is now `### Ambiguous prompt`, `## Large scope` is now `### Scope`, and `## Alternatives` moves under `## Before applying changes`. The branch step is the notable one: it previously reached the agent only as an aside in the signal protocol, so the doc's own instruction never shipped. + +`showMarkdownSecondary()` is emitted as the same `show-markdown` block as `showMarkdown()`, per the doc: for the MVP the two are equivalent. + +Also fixes the eco flags, which the sync would otherwise have broken silently (#314). They drop sections from the prompt by exact heading string, and the rewrite renames or moves every heading they matched; a missing heading is a no-op, so `--eco-auto-planning` and `--eco-auto-research` would have quietly stopped trimming anything, with no test failure to catch it. They are retargeted at `### Scope` and `### Alternatives`, the drop is now level-aware so removing a `###` stops at its sibling instead of swallowing the next `##`, and the tests assert a flag actually shortens the prompt rather than asserting a heading is absent (which passes for free once that heading is gone). + +`--eco-auto-maintenance` now drops nothing and is inert: the maintenance section left the system prompt for the post-merge prompt, so those tokens are already saved for every run. The flag stays parsed and persisted, and re-points at the post-merge prompt when that lands (#556). diff --git a/packages/framework/prompts/protocols/await.md b/packages/framework/prompts/protocols/await.md index 08154adb..1137f0aa 100644 --- a/packages/framework/prompts/protocols/await.md +++ b/packages/framework/prompts/protocols/await.md @@ -22,3 +22,4 @@ To display markdown in the side panel without blocking (a plan, a summary, a wri ``` This just shows it; you do not stop. Re-emit the same title to update that view in place. +showMarkdown() and showMarkdownSecondary() are both this block: emit the same `show-markdown` block for either. diff --git a/packages/framework/prompts/system_prompt.md b/packages/framework/prompts/system_prompt.md index 0c6b0f98..87ac0a5a 100644 --- a/packages/framework/prompts/system_prompt.md +++ b/packages/framework/prompts/system_prompt.md @@ -1,35 +1,60 @@ # System prompt -SHOW_MD: Show it to the user via `showMarkdown()` -SHOW_CHOICES: Show it to the user via `showChoices()` +SHOW_MD: Show it via `showMarkdown()` +SHOW_CHOICES: Show it via `showChoices()` AWAIT: Stop, await user answer before resuming -SESSION_NAME: the name of the current Git branch — sanitize it to be a SLUG, if name is generic (e.g. `main`) then create a succinct SLUG -SLUG: [a-z0-9-]+ +SESSION_NAME: the name of the session TODO_FILE: `TODO_.agent.md` +ADD_ANALYSIS_ENTRY: Add entry to the ANLYSIS_RESULT.md list -## Unclear scope +## Analyze the user prompt + +Analyze the user prompt, follow the instructions, create ANLYSIS_RESULT.md that lists the analysis results, and show it via showMarkdownSecondary() + +### Ambiguous prompt If it isn't clear what you should do (e.g. unclear scope, unclear user prompt), make a list of interpretations sorted by plausibility, , -## Large scope + whether YES/NO the prompt is ambiguous + +### Scope - If the scope of what you'll work on is *large*, create a `PLAN_.agent.md` of what you'll work on, , - If the scope is potentially *very large* (e.g. spans over many hours/days of work), also create a (backlog of follow-up tasks) and -## Alternatives + whether the scope is small, large, or very large + + +## Before starting changes + +Do the following before applying your first change. + +### Session name + +1. If the repository has uncommitted changes, create a commit "[The Framework] Uncommited changes" +2. Create a as a string [a-z0-9-]+ that succinctly represents the intention of the user prompt +3. Create a new branch `the-framework/` and `$ git checkout` it — do all the work in that branch +4. Call setSessionName() -Before starting to write code, measure "variability": -- List all high-level problems that need to be implemented -- Give a rating for each problem (from 0 to 10) following this criteria: is there an obviously optimal way to solve the problem (10), or is it highly unclear whether the problem can be solved in a better way (0)? + +## Before applying changes + +Do the following before applying changes — do it again anytime you make new changes. + +### Alternatives + +Measure "variability": +- List all high-level problems that you're about to solve +- Give a rating to each problem (from 0 to 10) following this criteria: is there an obviously optimal way to solve the problem (10), or is it highly unclear whether the problem can be solved in a better way (0)? - Explore and suggest alternatives for problems with a low rating - For each problem that has alternatives: list all alternatives sorted in a sensible order, , -## Maintenance -- When making changes to existing code, ${{ tf.params.autopilot ? "you can prefer minimal changes (e.g. to postpone a deep refactor)" : "prefer minimal changes to make it easier for humans to read the changes" }} -- But your changes should still be the correct solution on a high-level, don't implement a bad solution for the sake of making minimal changes -- If your changes aren't trivial and leads to refactor potential, add a new entry to - - The entry: "Look for refactoring opportunities arising from the merge" +## After applying changes + +After you're done, consider whether is finished and there isn't any work left to do — if that's the case then call setReadyForMerge() + + # User prompt diff --git a/packages/framework/src/prompt-run.test.ts b/packages/framework/src/prompt-run.test.ts index 7d7e6446..f13f0529 100644 --- a/packages/framework/src/prompt-run.test.ts +++ b/packages/framework/src/prompt-run.test.ts @@ -93,10 +93,10 @@ test('runPrompt honors eco flags in the emitted system prompt (#314)', async () }) const sys = events.find(e => e.kind === 'system-prompt') as { text: string } | undefined assert.ok(sys) - // The dropped sections are gone; the surviving one stays. - assert.ok(!sys.text.includes('## Alternatives')) - assert.ok(!sys.text.includes('## Maintenance')) - assert.ok(sys.text.includes('## Large scope')) + // The dropped section is gone; the surviving one stays. autoMaintenance drops nothing + // since #326 moved that section to the post-merge prompt (#556). + assert.ok(!sys.text.includes('### Alternatives')) + assert.ok(sys.text.includes('### Scope')) }) test('runPrompt with antiLazyPill false emits no built-in prompt even with eco set (#314)', async () => { @@ -114,7 +114,7 @@ test('runPrompt with antiLazyPill false emits no built-in prompt even with eco s assert.ok(sys) // Vanilla: no #326 block at all, only the always-on await protocol. assert.ok(!sys.text.includes('# System prompt')) - assert.ok(!sys.text.includes('## Unclear scope')) + assert.ok(!sys.text.includes('## Analyze the user prompt')) }) test('runPrompt pauses on a multi-select gate and continues with the pick (#331)', async () => { diff --git a/packages/framework/src/system-prompt.test.ts b/packages/framework/src/system-prompt.test.ts index 38fc05f6..e15a8c99 100644 --- a/packages/framework/src/system-prompt.test.ts +++ b/packages/framework/src/system-prompt.test.ts @@ -34,31 +34,45 @@ test('loadUserSystemPrompt is undefined when the file is absent or empty', async }) test('SYSTEM_PROMPT_TEMPLATE carries the #326 sections verbatim', () => { - for (const section of ['## Unclear scope', '## Large scope', '## Alternatives', '## Maintenance', '# User prompt']) { + for (const section of [ + '## Analyze the user prompt', + '### Ambiguous prompt', + '### Scope', + '## Before starting changes', + '### Session name', + '## Before applying changes', + '### Alternatives', + '## After applying changes', + '# User prompt', + ]) { assert.ok(SYSTEM_PROMPT_TEMPLATE.includes(section), `missing ${section}`) } assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('TODO_FILE: `TODO_.agent.md`')) + assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('ADD_ANALYSIS_ENTRY: Add entry to the ANLYSIS_RESULT.md list')) assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('${{tf.prompt}}')) - assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('${{ tf.params.autopilot ?')) + // The whole block is the branch-free doc now: #326 moved the one `tf.params.autopilot` + // ternary out with the maintenance section, so `tf.prompt` is the only fragment left. + assert.equal(SYSTEM_PROMPT_TEMPLATE.match(/\$\{\{/g)?.length, 1) +}) + +test('SYSTEM_PROMPT_TEMPLATE no longer carries the pre-#326-rewrite headings (#555)', () => { + // The 11-Jul draft's headings. They are what ECO_SECTION_HEADINGS used to match on, so + // if one comes back the eco mapping below is the thing to re-check. + for (const gone of ['## Unclear scope', '## Large scope', '## Maintenance']) { + assert.ok(!SYSTEM_PROMPT_TEMPLATE.includes(gone), `${gone} should be gone`) + } }) test('renderSystemPrompt splits the system and user halves', () => { const { system, user } = renderSystemPrompt({ prompt: 'build a todo app', params: {} }) assert.ok(system.startsWith('# System prompt')) - assert.ok(system.includes('## Maintenance')) + assert.ok(system.includes('## Analyze the user prompt')) + assert.ok(system.includes('## After applying changes')) assert.ok(!system.includes('# User prompt')) assert.ok(!system.includes('${{'), 'system half fully rendered') assert.equal(user, 'build a todo app') }) -test('renderSystemPrompt branches the maintenance line on tf.params.autopilot', () => { - const attended = renderSystemPrompt({ prompt: 'x', params: { autopilot: false } }).system - const autopilot = renderSystemPrompt({ prompt: 'x', params: { autopilot: true } }).system - assert.ok(attended.includes('prefer minimal changes to make it easier for humans to read the changes')) - assert.ok(autopilot.includes('you can prefer minimal changes (e.g. to postpone a deep refactor)')) - assert.ok(!autopilot.includes('easier for humans')) -}) - test('renderSystemPrompt is not confused by a user prompt containing the heading', () => { const sneaky = 'do X\n# User prompt\ndo Y' const { system, user } = renderSystemPrompt({ prompt: sneaky, params: {} }) @@ -103,56 +117,78 @@ test('systemPromptBlock ignores a whitespace-only user prompt', () => { }) test('systemPromptBlock threads tf through to the template', () => { - const block = systemPromptBlock({ tf: { prompt: 'x', params: { autopilot: true } } }) - assert.ok(block.includes('postpone a deep refactor')) + // `tf.prompt` lands in the user half, so eco is what observably reaches the system half. + const block = systemPromptBlock({ tf: { prompt: 'x', params: { eco: { autoResearch: true } } } }) + assert.ok(!block.includes('### Alternatives')) +}) + +test('every eco flag with a section still drops it (#314/#555)', () => { + // The regression this exists for: the mapping matches by exact heading string and + // dropSection() no-ops on a miss, so renaming a heading in the #326 doc silently stops a + // flag from trimming anything. A `!system.includes('## Large scope')` assertion cannot + // catch that, because it passes for free once the heading is gone. Assert the drop really + // shortens the prompt instead. + const full = renderSystemPrompt({ prompt: 'x', params: {} }).system + for (const flag of ['autoPlanning', 'autoResearch'] as const) { + const trimmed = renderSystemPrompt({ prompt: 'x', params: { eco: { [flag]: true } } }).system + assert.ok(trimmed.length < full.length, `eco.${flag} dropped nothing`) + } }) -test('eco.autoPlanning drops only the Large scope section (#314)', () => { +test('eco.autoPlanning drops only the Scope section (#314)', () => { const { system } = renderSystemPrompt({ prompt: 'x', params: { eco: { autoPlanning: true } } }) - assert.ok(!system.includes('## Large scope')) + assert.ok(!system.includes('### Scope')) assert.ok(!system.includes('PLAN_')) - // The neighbours survive, and the block spacing is intact. - assert.ok(system.includes('## Unclear scope')) - assert.ok(system.includes('## Alternatives')) - assert.ok(system.includes('## Maintenance')) - assert.ok(!system.includes('\n\n\n')) + assert.ok(!system.includes('whether the scope is small, large, or very large')) + // The `##` parent, the `###` sibling above it, and the section after it all survive: the + // drop stops at the next same-or-higher heading rather than running past it. + assert.ok(system.includes('## Analyze the user prompt')) + assert.ok(system.includes('### Ambiguous prompt')) + assert.ok(system.includes('whether YES/NO the prompt is ambiguous')) + assert.ok(system.includes('## Before starting changes')) + assert.ok(system.includes('### Alternatives')) }) test('eco.autoResearch drops only the Alternatives section (#314)', () => { const { system } = renderSystemPrompt({ prompt: 'x', params: { eco: { autoResearch: true } } }) - assert.ok(!system.includes('## Alternatives')) - assert.ok(!system.includes('measure "variability"')) - assert.ok(system.includes('## Large scope')) - assert.ok(system.includes('## Maintenance')) -}) - -test('eco.autoMaintenance drops the trailing Maintenance section cleanly (#314)', () => { + assert.ok(!system.includes('### Alternatives')) + assert.ok(!system.includes('Measure "variability"')) + // Its `##` parent stays, and so does the section after it. + assert.ok(system.includes('## Before applying changes')) + assert.ok(system.includes('## After applying changes')) + assert.ok(system.includes('### Scope')) +}) + +test('eco.autoMaintenance drops nothing: #326 moved the section to the post-merge prompt (#555/#556)', () => { + // Not a silent breakage but a deliberate no-op: the maintenance text left the system + // prompt, so there is nothing here to trim and the tokens are already saved for everyone. + // The flag stays parsed and persisted, and re-points at the post-merge prompt in #556. const { system, user } = renderSystemPrompt({ prompt: 'ship it', params: { eco: { autoMaintenance: true } } }) - assert.ok(!system.includes('## Maintenance')) - assert.ok(system.includes('## Alternatives')) - // The user half is untouched by eco, and no stray fragment survives the drop. - assert.equal(user, 'ship it') + assert.equal(system, renderSystemPrompt({ prompt: 'ship it', params: {} }).system) + assert.equal(user, 'ship it') // the user half is untouched by eco assert.ok(!system.includes('${{')) }) -test('all three eco drops leave just the Unclear scope section (#314)', () => { +test('both eco drops leave the rest of the prompt standing (#314)', () => { const { system } = renderSystemPrompt({ prompt: 'x', params: { eco: { autoPlanning: true, autoResearch: true, autoMaintenance: true } }, }) - assert.ok(system.includes('## Unclear scope')) - for (const gone of ['## Large scope', '## Alternatives', '## Maintenance']) { + for (const kept of ['## Analyze the user prompt', '### Ambiguous prompt', '### Session name', '## After applying changes']) { + assert.ok(system.includes(kept), `missing ${kept}`) + } + for (const gone of ['### Scope', '### Alternatives']) { assert.ok(!system.includes(gone), `${gone} should be dropped`) } }) test('no eco flags renders every section, and eco never touches the template', () => { const { system } = renderSystemPrompt({ prompt: 'x', params: { eco: {} } }) - for (const section of ['## Unclear scope', '## Large scope', '## Alternatives', '## Maintenance']) { + for (const section of ['## Analyze the user prompt', '### Scope', '### Alternatives', '## After applying changes']) { assert.ok(system.includes(section), `missing ${section}`) } // The living #326 doc stays byte-identical regardless of eco. - assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('## Large scope')) + assert.ok(SYSTEM_PROMPT_TEMPLATE.includes('### Scope')) }) test('vanilla (antiLazyPill false) wins over eco: no built-in prompt at all (#314)', () => { diff --git a/packages/framework/src/system-prompt.ts b/packages/framework/src/system-prompt.ts index ffbecf78..e94913e8 100644 --- a/packages/framework/src/system-prompt.ts +++ b/packages/framework/src/system-prompt.ts @@ -8,12 +8,13 @@ import { AWAIT_PROTOCOL, SIGNAL_PROTOCOL } from './turn-gate.js' // import here puts `node:fs` in the browser bundle. /** - * Rom's system prompt (#326), verbatim, as a template. It supersedes the - * anti-lazy-pill (#297/#301) it grew out of: unclear scope becomes a ranked - * `showChoices()` list, a large scope becomes a PLAN file to approve, a very - * large one also spins off a TODO backlog (consumed by the backlog loop, #323), - * the alternatives flow rates problem "variability" before code is written, and - * the maintenance section keeps edits minimal. + * The system prompt (#326), verbatim, as a template. It supersedes the + * anti-lazy-pill (#297/#301) it grew out of: the prompt is analyzed first into an + * ANLYSIS_RESULT.md, an ambiguous prompt becomes a ranked `showChoices()` list, a + * large scope becomes a PLAN file to approve, a very large one also spins off a TODO + * backlog (consumed by the backlog loop, #323), the work moves onto its own + * `the-framework/` branch before the first change, and the alternatives flow + * rates problem "variability" before code is written. * * Two layers make it executable: * - `${{ ... }}` fragments are JS evaluated against a {@link TfContext} (#350). @@ -30,25 +31,38 @@ import { AWAIT_PROTOCOL, SIGNAL_PROTOCOL } from './turn-gate.js' export const SYSTEM_PROMPT_TEMPLATE = SYSTEM_PROMPT /** - * Eco fine-grained control (#314): each flag drops one whole `##` section from the + * Eco fine-grained control (#314): each flag drops one whole section from the * built-in #326 prompt to save tokens, letting the agent auto-handle that concern. - * The template itself stays byte-identical to Rom's #326 doc; the sections are + * The template itself stays byte-identical to the #326 doc; the sections are * removed after the split, so a dropped section never leaves a fragment behind. */ export interface EcoOptions { - /** Drop `## Large scope` (the PLAN-file planning section). */ + /** Drop `### Scope` (the PLAN-file planning section). */ autoPlanning?: boolean | undefined - /** Drop `## Alternatives` (the variability-rating research section). */ + /** Drop `### Alternatives` (the variability-rating research section). */ autoResearch?: boolean | undefined - /** Drop `## Maintenance` (the minimal-changes maintenance section). */ + /** + * Drop the maintenance section. #326 moved it out of the system prompt and into the + * post-merge prompt, so there is nothing left here to trim and this flag currently + * drops nothing. It re-points at that prompt when it lands (#556); until then it stays + * parsed and persisted so the CLI flag and the dashboard toggle keep working. + */ autoMaintenance?: boolean | undefined } -/** Maps an {@link EcoOptions} flag to the `## ` heading it drops from the template. */ -const ECO_SECTION_HEADINGS: Record = { - autoPlanning: '## Large scope', - autoResearch: '## Alternatives', - autoMaintenance: '## Maintenance', +/** + * Maps an {@link EcoOptions} flag to the heading it drops from the template. Partial: a + * flag whose section no longer lives in this prompt has no entry. + * + * Every entry must match a heading that really exists. {@link dropSection} no-ops on a + * miss, so a heading rename in the #326 doc would silently stop the flag from trimming + * anything, with no test failure to catch it (a `!includes('## Large scope')` assertion + * passes for free once that heading is gone). `system-prompt.test.ts` pins each entry + * against the template and asserts the drop actually shortens the prompt. + */ +const ECO_SECTION_HEADINGS: Partial> = { + autoPlanning: '### Scope', + autoResearch: '### Alternatives', } /** The `tf` context the template's `${{...}}` fragments read (#326/#350). */ @@ -73,16 +87,22 @@ export interface RenderedSystemPrompt { const USER_PROMPT_HEADING = '\n# User prompt\n' /** - * Drop a whole `## ` section from a markdown block: everything from the - * heading up to (but not including) the next `## ` heading, or the end. The `\n\n` - * separator ahead of the section goes with it, so the surrounding blocks stay + * Drop a whole `` section from a markdown block: everything from the heading + * up to (but not including) the next heading of the same or a higher level, or the end. + * The `\n\n` separator ahead of the section goes with it, so the surrounding blocks stay * spaced exactly as before. A heading that isn't present is a no-op. + * + * Level-aware because #326 nests the eco-droppable sections under `##` parents: dropping + * `### Scope` has to stop at the next `###` sibling rather than run on to the next `##` + * and swallow it. */ function dropSection(md: string, heading: string): string { const at = md.indexOf(`\n${heading}`) if (at === -1) return md - const nextHeading = md.indexOf('\n## ', at + heading.length + 1) - const end = nextHeading === -1 ? md.length : nextHeading + const level = /^#+/.exec(heading)?.[0].length ?? 2 + const after = at + heading.length + 1 + const next = new RegExp(`\\n#{1,${level}} `).exec(md.slice(after)) + const end = next ? after + next.index : md.length return md.slice(0, at) + md.slice(end) } @@ -90,8 +110,8 @@ function dropSection(md: string, heading: string): string { function applyEco(systemHalf: string, eco: EcoOptions | undefined): string { if (!eco) return systemHalf let out = systemHalf - for (const key of Object.keys(ECO_SECTION_HEADINGS) as (keyof EcoOptions)[]) { - if (eco[key]) out = dropSection(out, ECO_SECTION_HEADINGS[key]) + for (const [key, heading] of Object.entries(ECO_SECTION_HEADINGS) as [keyof EcoOptions, string][]) { + if (eco[key]) out = dropSection(out, heading) } return out }