Add config-gated prompt templating

README.md

@@ -35,12 +35,51 @@ Fill in `.env` with the API key (and optional `*_BASE_URL`) for the provider you
 - `allowedModels` (optional) — picker list of `provider/model-id` refs (defaults to Anthropic, OpenAI, Gemini, and DeepSeek examples above)
 - `allowUserModelSelection` (optional) — when `true`, show a model picker and let the saved eval session override `model` with an entry from `allowedModels` (default `false`)
 - `allowCompare` (optional) — when `true`, show “Compare with another prompt” so learners can A/B two prompts (default `false`)
+- `features.promptTemplating` (optional) — enables reusable named blanks, shared examples, and a filled-in prompt preview. Missing configuration keeps the current Course 1 UI and `{{input}}` behavior unchanged.
 - `maxConcurrency` (optional) — max in-flight LLM calls during an evaluation (default `4`, range 1–50). Set to `1` for serial.
 - `defaults` (optional) — `runs` sets the initial run count while `minRuns`, `maxRuns`, `minCases`, and `maxCases` set the editable limits (each 1–5)
 - `initialSession` (optional) — `promptA`, `promptB`, and `cases` (`input` / `expectedAnswer`)
 
 Without `session.config.json`, prompts and cases start empty and the UI uses the built-in 1–5 limits. Copy `session.config.example.json` to prefill the capital-city demo.
 
+### Config-gated prompt templating
+
+Later-course tasks can let the prompt template define the fields shown in every case without changing Course 1:
+
+```json
+{
+  "allowCompare": false,
+  "features": {
+    "promptTemplating": {
+      "enabled": true,
+      "templateEditable": true,
+      "showPreview": true,
+      "dynamicFields": true,
+      "fields": [
+        { "name": "context", "label": "Context" },
+        { "name": "input", "label": "Input" },
+        { "name": "constraint", "label": "Constraint" }
+      ]
+    }
+  },
+  "initialSession": {
+    "promptA": "Use the context to answer the input.\n\nContext:\n{{context}}\n\nInput:\n{{input}}\n\nConstraint:\n{{constraint}}",
+    "cases": [
+      {
+        "input": "",
+        "expectedAnswer": "",
+        "variables": {
+          "context": "",
+          "constraint": ""
+        }
+      }
+    ]
+  }
+}
+```
+
+With `dynamicFields`, placeholders such as `{{context}}`, `{{input}}`, and `{{constraint}}` automatically become labeled fields in each case. Learners edit the template as normal text, while placeholder values can differ across cases. Text written directly in the template stays shared. Expected output is used only for scoring, and each case can show its exact rendered prompt.
+
 Work-in-progress (prompts, cases, settings, and the last results) is stored in `eval-session.json`. That file is local and not checked in. A saved session wins over `initialSession` on reload.
 
 After each evaluation run, the server appends a numbered `Evaluation` section to `.codesignal/report.md`. The report keeps all evaluations from the current workspace, including each evaluation's setup, overall scores, cases, and individual runs. That file is gitignored.

lib/eval-compare.js

@@ -7,7 +7,7 @@
 import { createConcurrencyLimiter, normalizeConcurrency } from './concurrency.js';
 import { DEFAULT_METRIC_ID, summarizeScores } from './metrics/index.js';
 import { normalizeRunCount, runEvalBatch } from './eval-run.js';
-import { renderPromptTemplate } from './prompt-render.js';
+import { findPromptPlaceholders, renderPromptTemplate } from './prompt-render.js';
 
 export const MIN_EVAL_CASES = 1;
 export const MAX_EVAL_CASES = 5;
@@ -25,6 +25,7 @@ export const MAX_EVAL_CASES = 5;
  * @property {string} [label]
  * @property {string} input
  * @property {string} [expectedAnswer]
+ * @property {Record<string, string>} [variables]
  */
 
 /**
@@ -64,10 +65,30 @@ export function compareByMean(promptResults) {
  *   cases?: EvalCase[],
  *   input?: string,
  *   expectedAnswer?: string,
+ *   prompts?: PromptVariant[],
+ *   promptTemplating?: { enabled?: boolean, allowExamples?: boolean, variableNames?: string[], dynamicFields?: boolean },
  * }} opts
- * @returns {Array<{ id: string, label: string, input: string, expectedAnswer: string }>}
+ * @returns {Array<{
+ *   id: string,
+ *   label: string,
+ *   input: string,
+ *   expectedAnswer: string,
+ *   variables?: Record<string, string>,
+ * }>}
  */
 export function normalizeCases(opts = {}) {
+  const templatingEnabled = opts.promptTemplating?.enabled === true;
+  const variableNames = templatingEnabled
+    ? (opts.promptTemplating?.dynamicFields
+      ? [...new Set((opts.prompts ?? []).flatMap((prompt) => (
+        findPromptPlaceholders(prompt?.promptTemplate)
+      )))].filter((name) => (
+        !['input', 'examples', '__proto__', 'prototype', 'constructor'].includes(name)
+      ))
+      : (Array.isArray(opts.promptTemplating?.variableNames)
+        ? opts.promptTemplating.variableNames
+        : []))
+    : [];
   if (Array.isArray(opts.cases) && opts.cases.length > 0) {
     if (opts.cases.length > MAX_EVAL_CASES) {
       const err = new Error(`At most ${MAX_EVAL_CASES} test cases are allowed`);
@@ -95,6 +116,14 @@ export function normalizeCases(opts = {}) {
         label: typeof c.label === 'string' && c.label.trim() ? c.label.trim() : `Case ${i + 1}`,
         input: typeof c.input === 'string' ? c.input : '',
         expectedAnswer: typeof c.expectedAnswer === 'string' ? c.expectedAnswer : '',
+        ...(templatingEnabled
+          ? {
+            variables: Object.fromEntries(variableNames.map((name) => [
+              name,
+              typeof c.variables?.[name] === 'string' ? c.variables[name] : '',
+            ])),
+          }
+          : {}),
       };
     });
   }
@@ -105,6 +134,7 @@ export function normalizeCases(opts = {}) {
       label: 'Case 1',
       input: typeof opts.input === 'string' ? opts.input : '',
       expectedAnswer: typeof opts.expectedAnswer === 'string' ? opts.expectedAnswer : '',
+      ...(templatingEnabled ? { variables: {} } : {}),
     },
   ];
 }
@@ -143,12 +173,18 @@ function validatePrompts(prompts) {
 /**
  * Render every prompt × case and reject before any batch work if one is empty.
  * @param {PromptVariant[]} prompts
- * @param {Array<{ input: string }>} cases
+ * @param {Array<{ input: string, variables?: Record<string, string> }>} cases
+ * @param {Array<{ input?: string, idealOutput?: string }>} examples
+ * @param {{ enabled?: boolean }} promptTemplating
  */
-function assertRenderedPrompts(prompts, cases) {
+function assertRenderedPrompts(prompts, cases, examples, promptTemplating) {
   for (const testCase of cases) {
     for (const p of prompts) {
-      const renderedPrompt = renderPromptTemplate(p.promptTemplate, testCase.input);
+      const renderedPrompt = renderPromptTemplate(p.promptTemplate, testCase.input, {
+        variables: testCase.variables,
+        examples,
+        strict: promptTemplating?.enabled === true,
+      });
       if (!renderedPrompt.trim()) {
         const err = new Error('promptTemplate (or input) must produce a non-empty prompt');
         err.code = 'EMPTY_PROMPT';
@@ -167,6 +203,8 @@ function assertRenderedPrompts(prompts, cases) {
  *   cases?: EvalCase[],
  *   input?: string,
  *   expectedAnswer?: string,
+ *   examples?: Array<{ input?: string, idealOutput?: string }>,
+ *   promptTemplating?: { enabled?: boolean, variableNames?: string[] },
  *   metricId?: string,
  *   runs?: number,
  *   maxConcurrency?: number,
@@ -188,9 +226,20 @@ export async function runPromptComparison(deps, opts) {
   const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : null;
   const compareMode = prompts.length > 1;
   const total = cases.length * prompts.length * runs;
+  const examples = opts.promptTemplating?.enabled === true
+    && opts.promptTemplating?.allowExamples === true
+    && Array.isArray(opts.examples)
+    ? opts.examples
+      .filter((example) => example && typeof example === 'object')
+      .map((example) => ({
+        input: typeof example.input === 'string' ? example.input : '',
+        idealOutput: typeof example.idealOutput === 'string' ? example.idealOutput : '',
+      }))
+      .slice(0, 5)
+    : [];
 
   const anyExpected = cases.some((c) => c.expectedAnswer.trim() !== '');
-  assertRenderedPrompts(prompts, cases);
+  assertRenderedPrompts(prompts, cases, examples, opts.promptTemplating);
   const startedAt = Date.now();
   const conditions = {
     metricId: anyExpected ? metricId : null,
@@ -221,6 +270,9 @@ export async function runPromptComparison(deps, opts) {
           const batch = await runBatch(deps, {
             promptTemplate: p.promptTemplate,
             input: testCase.input,
+            templateVariables: testCase.variables,
+            examples,
+            strictTemplating: opts.promptTemplating?.enabled === true,
             expectedAnswer: testCase.expectedAnswer,
             metricId,
             runs,
@@ -281,6 +333,7 @@ export async function runPromptComparison(deps, opts) {
       label: testCase.label,
       input: testCase.input,
       expectedAnswer: testCase.expectedAnswer.trim() !== '' ? testCase.expectedAnswer : null,
+      ...(opts.promptTemplating?.enabled ? { variables: testCase.variables } : {}),
       prompts: promptResults,
       comparison: compareByMean(promptResults),
     };
@@ -311,6 +364,9 @@ export async function runPromptComparison(deps, opts) {
       ...conditions,
       durationMs: Math.max(0, Date.now() - startedAt),
     },
+    ...(opts.promptTemplating?.enabled
+      ? { templateContext: { examples } }
+      : {}),
     cases: caseResults,
     prompts: overallPrompts,
     comparison: compareByMean(overallPrompts),

lib/eval-report.js

@@ -49,15 +49,18 @@ function clip(text, max = 240) {
  *     conditions?: { metricId?: string | null, runs?: number, caseCount?: number, maxConcurrency?: number, durationMs?: number },
  *     comparison?: { outcome?: string, winnerId?: string | null, means?: Record<string, number | null> },
  *     prompts?: Array<{ id: string, label?: string, aggregate?: { mean?: number, min?: number, max?: number, count?: number } | null }>,
+ *     templateContext?: { examples?: Array<{ input?: string, idealOutput?: string }> },
  *     cases?: Array<{
  *       id?: string,
  *       label?: string,
  *       input?: string,
  *       expectedAnswer?: string | null,
+ *       variables?: Record<string, string>,
  *       comparison?: { outcome?: string, winnerId?: string | null, means?: Record<string, number | null> },
  *       prompts?: Array<{
  *         id: string,
  *         label?: string,
+ *         renderedPrompt?: string,
  *         aggregate?: { mean?: number } | null,
  *         results?: Array<{ run?: number, status?: string, score?: number | null, output?: string, error?: string | null }>,
  *       }>,
@@ -73,6 +76,9 @@ export function buildEvalReportMarkdown(opts) {
   const comparison = result.comparison && typeof result.comparison === 'object' ? result.comparison : {};
   const prompts = Array.isArray(result.prompts) ? result.prompts : [];
   const cases = Array.isArray(result.cases) ? result.cases : [];
+  const examples = Array.isArray(result.templateContext?.examples)
+    ? result.templateContext.examples
+    : [];
   const generatedAt = opts.generatedAt || new Date().toISOString();
   const multi = prompts.length > 1;
 
@@ -115,6 +121,19 @@ export function buildEvalReportMarkdown(opts) {
     );
   }
 
+  if (examples.length > 0) {
+    lines.push('## Shared examples', '');
+    examples.forEach((example, index) => {
+      lines.push(
+        `### Example ${index + 1}`,
+        '',
+        `- **Input:** ${escapeCell(example.input || '(empty)')}`,
+        `- **Ideal output:** ${escapeCell(example.idealOutput || '(empty)')}`,
+        '',
+      );
+    });
+  }
+
   lines.push('## Overall results', '');
 
   if (comparison.outcome === 'winner' && comparison.winnerId) {
@@ -152,6 +171,11 @@ export function buildEvalReportMarkdown(opts) {
         ? '_none_'
         : escapeCell(testCase.expectedAnswer)}`,
     );
+    if (testCase.variables && typeof testCase.variables === 'object') {
+      for (const [name, value] of Object.entries(testCase.variables)) {
+        lines.push(`- **${escapeCell(name)}:** ${escapeCell(value || '(empty)')}`);
+      }
+    }
     if (multi && testCase.comparison) {
       if (testCase.comparison.outcome === 'winner' && testCase.comparison.winnerId) {
         lines.push(`- **Case winner:** Prompt ${escapeCell(testCase.comparison.winnerId)}`);
@@ -165,6 +189,16 @@ export function buildEvalReportMarkdown(opts) {
 
     for (const prompt of testCase.prompts || []) {
       lines.push(`#### ${escapeCell(prompt.label || prompt.id)}`, '');
+      if (typeof prompt.renderedPrompt === 'string') {
+        lines.push(
+          'Filled-in prompt:',
+          '',
+          '```',
+          prompt.renderedPrompt.trim() || '(empty)',
+          '```',
+          '',
+        );
+      }
       lines.push(`Mean: ${formatScore(prompt.aggregate?.mean)}`, '');
       lines.push('| Run | Status | Score | Output |');
       lines.push('| --- | --- | --- | --- |');

lib/eval-run.js

@@ -77,6 +77,9 @@ export async function resolveSystemPrompt(explicit) {
  *   run: number,
  *   expectedAnswer?: string,
  *   metricId?: string,
+ *   templateVariables?: Record<string, string>,
+ *   examples?: Array<{ input?: string, idealOutput?: string }>,
+ *   strictTemplating?: boolean,
  * }} opts
  * @returns {Promise<EvalResult>}
  */
@@ -137,7 +140,11 @@ export async function runEvalBatch(deps, opts) {
   const expectedAnswer = typeof opts.expectedAnswer === 'string' ? opts.expectedAnswer : '';
   const metricId = opts.metricId ?? DEFAULT_METRIC_ID;
   const runs = normalizeRunCount(opts.runs);
-  const renderedPrompt = renderPromptTemplate(promptTemplate, input);
+  const renderedPrompt = renderPromptTemplate(promptTemplate, input, {
+    variables: opts.templateVariables,
+    examples: opts.examples,
+    strict: opts.strictTemplating === true,
+  });
   const schedule = typeof opts.schedule === 'function'
     ? opts.schedule
     : createConcurrencyLimiter(opts.maxConcurrency).schedule;