fix(core): harden the eval harness from live A/B and BEAM runs

benchmarks/src/basic_memory_benchmarks/agent_tasks/driver.py

@@ -258,21 +258,29 @@ def _prepare_task_project(
     question is untenable, and grouped runs are read-only so reuse cannot leak
     state between tasks.
     """
+    # The CLI already prefixes generated run ids with "at-"; prepending another
+    # literal here produced "at-at-..." project names in the first real run.
     if task.group is None:
         project = TaskProject(
-            name=f"at-{run_id}-{task.id}", directory=surface_home / "projects" / task.id
+            name=f"{run_id}-{task.id}", directory=surface_home / "projects" / task.id
         )
         source_dir = corpus_dir
     else:
         cached = prepared_groups.get(task.group)
         if cached is not None:
             return cached
         project = TaskProject(
-            name=f"at-{run_id}-{task.group}", directory=surface_home / "projects" / task.group
+            name=f"{run_id}-{task.group}", directory=surface_home / "projects" / task.group
         )
         source_dir = corpus_dir / task.group / "docs"
     copy_corpus(source_dir, project.directory)
     run_command(prefix + ["project", "add", project.name, str(project.directory)], env=env)
+    # `project add` registers but does not index: without an explicit index
+    # pass the DB stays empty, `status` reports ready vacuously (zero pending
+    # work was ever queued), and every retrieval tool sees an empty project.
+    # The scripted smoke masked this — its canned answers never needed the
+    # index — so the first real-model run surfaced it as 24 empty projects.
+    run_command(prefix + ["reindex", "-p", project.name, "--full", "--search"], env=env)
     settle_index(
         prefix=prefix,
         env=env,
@@ -321,6 +329,16 @@ def dispatch(name: str, arguments: dict[str, Any]) -> ToolOutcome:
         budget=config.budget,
     )
     if needs_state:
+        # Trigger: the task is graded on project state (files, SQLite index).
+        # Why: a wikilink written mid-loop lands as an unresolved relation row;
+        # BM resolves forward references in a later index pass, and settle only
+        # watches file-sync work — grading straight after settle raced that
+        # pass (first real run: rich curate-connect failed RelationResolves
+        # despite a correct, project-scoped edit_note).
+        # Outcome: re-run the project index (its completion step resolves
+        # forward references — same pass _prepare_task_project relies on),
+        # then settle, so graders read converged state.
+        run_command(prefix + ["reindex", "-p", project.name, "--search"], env=env)
         settle_index(
             prefix=prefix,
             env=env,
@@ -549,7 +567,7 @@ def _write_jsonl(path: Path, rows: list[dict]) -> None:
 def run_agent_tasks(
     config: AgentTasksConfig,
     *,
-    model_factory: Callable[[str], ToolAgentModel] = create_tool_agent_model,
+    model_factory: Callable[[str], ToolAgentModel] | None = None,
     session_factory: Callable[[SurfaceRuntime], AgentSession] | None = None,
     judge_factory: Callable[[str], LLMRunner] = create_runner,
 ) -> Path:
@@ -598,7 +616,19 @@ def run_agent_tasks(
 
     # Model and judge parse before any on-disk state is created, so a bad spec
     # fails fast without leaving an empty benchmark home behind.
-    model = model_factory(config.model_spec)
+    # Trigger: a programmatic caller builds AgentTasksConfig and calls this
+    #   directly, rather than through the CLI, which pre-binds temperature.
+    # Why: manifest.json records config.model_temperature, so constructing the
+    #   model without it would run at the factory default while the artifact
+    #   claimed the configured value — a silent provenance lie.
+    # Outcome: the default path carries the recorded temperature; an injected
+    #   factory owns its own configuration and is called unchanged. The default
+    #   is a None sentinel rather than the function itself so the branch reads
+    #   the module attribute at call time, which is also what makes it testable.
+    if model_factory is None:
+        model = create_tool_agent_model(config.model_spec, temperature=config.model_temperature)
+    else:
+        model = model_factory(config.model_spec)
     judge = judge_factory(config.judge_spec) if config.judge_spec else None
     build_session = session_factory or (lambda runtime: McpAgentSession(runtime))
 

benchmarks/src/basic_memory_benchmarks/agent_tasks/grading.py

@@ -108,6 +108,17 @@ def normalize_answer_item(value: str) -> str:
     return value.strip().lstrip("/").removesuffix(".md").lower()
 
 
+def strip_own_project_prefix(item: str, project_name: str) -> str:
+    """Drop the task's OWN project-name prefix from a normalized answer item.
+
+    Agents quote permalinks exactly as tools return them — ``<project>/<permalink>``
+    — while gold values are project-relative. Only this task's project name is
+    stripped: an item carrying a DIFFERENT project's prefix is genuinely wrong
+    (cross-project leakage) and must keep failing.
+    """
+    return item.removeprefix(normalize_answer_item(project_name) + "/")
+
+
 # --- File helpers ---
 
 
@@ -193,7 +204,7 @@ def _eval_answer_set(grader: AnswerSetEquals, ctx: GradingContext) -> PredicateR
     raw = payload.get(grader.key)
     if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw):
         return _result(grader, False, f"answer key '{grader.key}' is not a list of strings")
-    got = {normalize_answer_item(item) for item in raw}
+    got = {strip_own_project_prefix(normalize_answer_item(item), ctx.project_name) for item in raw}
     gold = {normalize_answer_item(item) for item in grader.gold}
     if got == gold:
         return _result(grader, True, f"{len(gold)} items match")
@@ -271,21 +282,31 @@ def _eval_relation_resolves(grader: RelationResolves, ctx: GradingContext) -> Pr
         if row is None:
             raise RuntimeError(f"Project '{ctx.project_name}' not found in {ctx.db_path}")
         project_id = int(row[0])
+        # Stored permalinks are project-prefixed (verified against a live run
+        # DB: 'at-<run>-<task>/notes/redis-cache-tuning'), while task specs
+        # use project-relative gold — match either form, same policy as
+        # strip_own_project_prefix for answer-set graders.
         query = (
             "SELECT target.permalink, r.relation_type"
             " FROM relation r"
             " JOIN entity source ON r.from_id = source.id"
             " JOIN entity target ON r.to_id = target.id"
-            " WHERE source.project_id = ? AND source.permalink = ?"
+            " WHERE source.project_id = ? AND source.permalink IN (?, ?)"
             " AND r.to_id IS NOT NULL"
         )
-        rows = connection.execute(query, (project_id, grader.source_permalink)).fetchall()
+        prefixed_source = f"{ctx.project_name}/{grader.source_permalink}"
+        rows = connection.execute(
+            query, (project_id, grader.source_permalink, prefixed_source)
+        ).fetchall()
     finally:
         connection.close()
 
     targets = {normalize_answer_item(item) for item in grader.targets}
     for target_permalink, relation_type in rows:
-        if normalize_answer_item(str(target_permalink)) not in targets:
+        resolved = strip_own_project_prefix(
+            normalize_answer_item(str(target_permalink)), ctx.project_name
+        )
+        if resolved not in targets:
             continue
         if grader.relation_type is not None and relation_type != grader.relation_type:
             continue

benchmarks/src/basic_memory_benchmarks/agent_tasks/models.py

@@ -2,16 +2,25 @@
 
 from __future__ import annotations
 
+import re
 from typing import Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 from basic_memory_benchmarks.models import RuntimeInfo
 
 # "error" marks a loop that died mid-task (model or dispatch failure): the
 # partial accounting is kept on the errored row, never counted as a budget stop.
 StopReason = Literal["final", "turns", "tokens", "wall_clock", "error"]
 
+# run_id is not merely a label. The driver splices it into a `bm project add`
+# argv token (project names are `{run_id}-{task.id}`) and into two filesystem
+# paths (the benchmark home and the run dir), so it must be safe as both. A
+# leading "-" makes the BM CLI parse the project name as options and abort with
+# "No such option: -t" — naming a flag the operator never typed — and a path
+# separator would place run artifacts outside the run dir.
+RUN_ID_PATTERN = re.compile(r"[A-Za-z0-9_][A-Za-z0-9._-]*")
+
 
 class AgentBudget(BaseModel):
     """Per-task budgets; identical across surfaces (fairness contract)."""
@@ -29,6 +38,15 @@ class AgentTasksConfig(BaseModel):
     # task set. Manifest runs are grouped and read-only (see driver).
     task_manifest: str | None = None
     model_spec: str
+    # None = the temperature parameter is omitted from requests entirely
+    # (Claude 5 endpoints reject it). Provenance: the value the model factory
+    # used for this run.
+    # allow_inf_nan=False because nan/inf survive float() and Pydantic's plain
+    # float schema: they would reach the request body, where httpx raises a bare
+    # ValueError mid-run. Worse for provenance, model_dump_json() encodes them
+    # as null — which is exactly this field's "temperature omitted" sentinel, so
+    # the recorded run config would silently misreport what was sent.
+    model_temperature: float | None = Field(default=0.0, allow_inf_nan=False)
     judge_spec: str | None = None
     corpus_dir: str = "benchmarks/datasets/agent-tasks/corpus"
     output_root: str = "benchmarks/runs"
@@ -39,6 +57,23 @@ class AgentTasksConfig(BaseModel):
     settle_timeout_seconds: float = Field(default=180.0, gt=0.0)
     allow_surface_skip: bool = True
 
+    @field_validator("run_id")
+    @classmethod
+    def _run_id_is_argv_and_path_safe(cls, value: str) -> str:
+        # Rejected here rather than in the driver so the run dies before any
+        # setup cost: without this the first `bm project add` fails only after
+        # the benchmark home, a warm `bm mcp` subprocess, and a full corpus copy
+        # exist — and run_command captures stderr, so the operator sees a bare
+        # CalledProcessError exit status, not the CLI's own complaint. The
+        # abandoned home then blocks re-running the same run_id.
+        if not RUN_ID_PATTERN.fullmatch(value):
+            raise ValueError(
+                "run_id must start with a letter, digit, or underscore and use only "
+                "letters, digits, '.', '_', or '-'; it becomes both a CLI argument "
+                f"and a path component, got {value!r}"
+            )
+        return value
+
 
 class TurnRecord(BaseModel):
     """One model turn or one tool dispatch inside a task's agent loop."""

benchmarks/src/basic_memory_benchmarks/cli.py

@@ -3,11 +3,14 @@
 from __future__ import annotations
 
 import json
+import math
 import shutil
 import uuid
+from functools import partial
 from pathlib import Path
 
 import typer
+from pydantic import ValidationError
 from rich.console import Console
 
 from basic_memory_benchmarks.agent_tasks.driver import run_agent_tasks
@@ -465,6 +468,21 @@ def run_agent_tasks_command(
         "--model",
         help="Agent under test: openai-compat:<model>@<base_url> | scripted:<path.json>",
     ),
+    model_header: list[str] | None = typer.Option(
+        None,
+        "--model-header",
+        help="Extra HTTP header for the agent endpoint as 'Name=value' (repeatable); "
+        "e.g. anthropic-workspace-id=wrkspc_... for identity-linked Anthropic keys. "
+        "Header names are matched case-insensitively, so 'authorization=...' "
+        "replaces the bearer derived from OPENAI_API_KEY rather than joining it. "
+        "Values are never recorded in run artifacts.",
+    ),
+    model_temperature: str = typer.Option(
+        "0",
+        "--model-temperature",
+        help="Sampling temperature for the agent endpoint, or 'omit' to send none "
+        "(Claude 5 models reject the parameter). Recorded in the run config.",
+    ),
     judge: str | None = typer.Option(
         None,
         "--judge",
@@ -535,10 +553,38 @@ def run_agent_tasks_command(
     except (ValueError, FileNotFoundError) as exc:
         raise typer.BadParameter(str(exc)) from exc
 
+    # Extra endpoint headers stay out of AgentTasksConfig (and therefore out
+    # of run artifacts): values may be sensitive, so they ride only in the
+    # model factory closure below.
+    header_pairs: dict[str, str] = {}
+    for raw_header in model_header or []:
+        name, separator, value = raw_header.partition("=")
+        if not separator or not name.strip() or not value.strip():
+            raise typer.BadParameter(f"--model-header must be 'Name=value', got {raw_header!r}")
+        header_pairs[name.strip()] = value.strip()
+
+    if model_temperature.strip().lower() in {"omit", "none"}:
+        temperature: float | None = None
+    else:
+        try:
+            temperature = float(model_temperature)
+        except ValueError:
+            raise typer.BadParameter(
+                f"--model-temperature must be a number or 'omit', got {model_temperature!r}"
+            ) from None
+        # nan/inf parse cleanly and pass config validation, but JSON has no
+        # encoding for them: httpx raises a bare ValueError when it serializes
+        # the request body. That escapes _post's handled transports, so the run
+        # dies mid-flight after surface setup with no artifacts written.
+        if not math.isfinite(temperature):
+            raise typer.BadParameter(
+                f"--model-temperature must be a finite number or 'omit', got {model_temperature!r}"
+            )
+
     # Fail fast at parse time: a bad model spec (including claude:) and a
     # missing judge for judge-graded tasks must not survive to mid-run.
     try:
-        create_tool_agent_model(model)
+        create_tool_agent_model(model, extra_headers=header_pairs or None, temperature=temperature)
     except ValueError as exc:
         raise typer.BadParameter(str(exc)) from exc
     judged = [
@@ -550,27 +596,41 @@ def run_agent_tasks_command(
         raise typer.BadParameter(f"Tasks {judged} use judge_rubric graders; pass --judge")
 
     resolved_run_id = run_id or f"at-{uuid.uuid4().hex[:12]}"
-    config = AgentTasksConfig(
-        run_id=resolved_run_id,
-        surfaces=surface_list,
-        task_ids=task_ids,
-        task_manifest=str(task_manifest) if task_manifest is not None else None,
-        model_spec=model,
-        judge_spec=judge,
-        corpus_dir=str(corpus_dir),
-        output_root=str(output_root),
-        bm_source=bm_source,
-        bm_local_path=str(bm_local_path),
-        budget=AgentBudget(
-            max_turns=max_turns,
-            max_total_tokens=max_total_tokens,
-            max_task_seconds=task_timeout,
+    # AgentTasksConfig owns the field rules (finite temperature, argv/path-safe
+    # run_id) so the direct run_agent_tasks(config) path is guarded too. Render
+    # its rejection as a CLI parameter error instead of a Pydantic traceback.
+    try:
+        config = AgentTasksConfig(
+            run_id=resolved_run_id,
+            surfaces=surface_list,
+            task_ids=task_ids,
+            task_manifest=str(task_manifest) if task_manifest is not None else None,
+            model_spec=model,
+            model_temperature=temperature,
+            judge_spec=judge,
+            corpus_dir=str(corpus_dir),
+            output_root=str(output_root),
+            bm_source=bm_source,
+            bm_local_path=str(bm_local_path),
+            budget=AgentBudget(
+                max_turns=max_turns,
+                max_total_tokens=max_total_tokens,
+                max_task_seconds=task_timeout,
+            ),
+            tool_timeout_seconds=tool_timeout,
+            settle_timeout_seconds=settle_timeout,
+            allow_surface_skip=allow_surface_skip,
+        )
+    except ValidationError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    run_dir = run_agent_tasks(
+        config,
+        model_factory=partial(
+            create_tool_agent_model,
+            extra_headers=header_pairs or None,
+            temperature=temperature,
         ),
-        tool_timeout_seconds=tool_timeout,
-        settle_timeout_seconds=settle_timeout,
-        allow_surface_skip=allow_surface_skip,
     )
-    run_dir = run_agent_tasks(config)
     console.print(f"Agent-task run complete: [green]{run_dir}[/green]")
     console.print(f"See [cyan]{run_dir / 'summary.md'}[/cyan]")
 

benchmarks/src/basic_memory_benchmarks/converters/beam_to_corpus.py

@@ -42,11 +42,17 @@
 BEAM_CITATION = "BEAM: Beyond a Million Tokens (ICLR 2026, arXiv 2510.27246)"
 BEAM_LICENSE_NOTE = "Code MIT; benchmark data CC BY-SA 4.0"
 
-# Trailing probe-index marker in message content (e.g. "... ->-> 1,2"). The
-# second field is not always numeric — upstream 100K/1/chat.json ends one
-# message with "->-> 2,N/A" — so both fields match any non-space token. It
-# links transcript text back to probe indices, so it must never be ingested.
-_INDEX_MARKER_PATTERN = re.compile(r"\s*->->\s*\S+,\S+\s*$")
+# Trailing probe-index marker in message content: "->->" followed by a
+# comma-separated id list. A survey of every marker in the 100K tier
+# (2,199 total) found exactly five shapes: " 1,1" (2190), " 1,5)" (6),
+# " 2,N/A" (1), " 2,22, 24" (1), and a double-space variant (1). The
+# first id is always an int; later ids are ints or N/A, spaces around
+# commas optional; the trailing ")" is generator junk (every such message
+# contains zero opening parens), so it strips with the marker. Kept this
+# narrow deliberately — anything else containing "->->" trips the
+# fail-fast below instead of silently stripping content. The marker links
+# transcript text back to probe indices, so it must never be ingested.
+_INDEX_MARKER_PATTERN = re.compile(r"\s*->->\s*\d+(?:\s*,\s*(?:\d+|N/A))*\)?\s*$")
 
 
 @dataclass(frozen=True)