From 41f876ab7caeb303757243487b00ff82096c3153 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Mon, 10 Aug 2026 10:32:53 +0000 Subject: [PATCH 1/3] bench(concurrency): add ragged paired benchmark --- harness/benchmarks/README.md | 3 + harness/benchmarks/concurrency/README.md | 69 ++++ .../concurrency/concurrent_benchmark.py | 349 ++++++++++++++++++ .../concurrency/generate_ragged_prompts.py | 78 ++++ .../concurrency/run_qwen36_concurrency.sh | 216 +++++++++++ .../concurrency/summarize_concurrency.py | 188 ++++++++++ .../concurrency/test_concurrency_tools.py | 202 ++++++++++ .../concurrency/test_concurrent_benchmark.py | 124 +++++++ 8 files changed, 1229 insertions(+) create mode 100644 harness/benchmarks/concurrency/README.md create mode 100755 harness/benchmarks/concurrency/concurrent_benchmark.py create mode 100755 harness/benchmarks/concurrency/generate_ragged_prompts.py create mode 100755 harness/benchmarks/concurrency/run_qwen36_concurrency.sh create mode 100755 harness/benchmarks/concurrency/summarize_concurrency.py create mode 100644 harness/benchmarks/concurrency/test_concurrency_tools.py create mode 100644 harness/benchmarks/concurrency/test_concurrent_benchmark.py diff --git a/harness/benchmarks/README.md b/harness/benchmarks/README.md index c52d21cfb..722e875d9 100644 --- a/harness/benchmarks/README.md +++ b/harness/benchmarks/README.md @@ -4,6 +4,9 @@ These checks are separate from the client harness launchers. They compare Lucebo generation against a llama.cpp baseline on the same target GGUF, using small deterministic prompts. +For the paired ragged C1/C4/C8/C16 serving benchmark, see +[`concurrency/`](concurrency/README.md). + Use this when you want to know whether a server change affects output quality or decode speed. Use `harness/clients/` when you want to know whether Codex, OpenCode, Open WebUI, Pi, and the other clients still work. diff --git a/harness/benchmarks/concurrency/README.md b/harness/benchmarks/concurrency/README.md new file mode 100644 index 000000000..c310251f5 --- /dev/null +++ b/harness/benchmarks/concurrency/README.md @@ -0,0 +1,69 @@ +# Qwen3.6 concurrency benchmark + +This protocol measures the serving behavior targeted by packed continuous +prefill and concurrent decode. It is intentionally small: one streaming client, +one fresh-process runner, one deterministic prompt generator, and one summary +script. + +Run a quick screening repeat: + +```bash +MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +LUCE_SERVER_BIN=server/build-hip/dflash_server \ +LLAMA_SERVER_BIN=/path/to/llama-server \ +harness/benchmarks/concurrency/run_qwen36_concurrency.sh +``` + +Run a decode-heavy comparison with the same harness: + +```bash +MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +LUCE_SERVER_BIN=server/build-hip/dflash_server \ +LLAMA_SERVER_BIN=/path/to/llama-server \ +WORKLOADS=short MAX_TOKENS=256 VARIANTS=luce-k8,llama REPEATS=3 \ +harness/benchmarks/concurrency/run_qwen36_concurrency.sh +``` + +The short ragged prompts keep admission realistic while 256 forced output +tokens make generation dominate the measured window. Use `REPEATS=5` for +publication. Every measured case starts a fresh server and first runs a +discarded warmup at the same concurrency. The variants are: + +- `luce-k8`: packed prefill with up to eight concurrent prefills. +- `luce-k1`: the same binary/configuration with packing width limited to one. +- `llama`: llama.cpp continuous batching with fixed `-b 2048 -ub 512`. + +The 29 generated prompts are disjoint cohorts for C1/C4/C8/C16. C4 and above +contain four substantial length strata while holding the mean target length +constant. The default short, medium, and long profiles target approximately +400, 1,000, and 3,000 input tokens per request. The client refuses to wrap or +reuse a prompt; reports retain the exact server-observed token counts. + +The headline metric is aggregate output goodput: exact server-reported +completion tokens divided by level wall time. It includes queueing, prefill, +and decode and must not be called decode throughput. + +`Output-window tok/s` divides exact completion tokens by the interval from the +earliest observed first output to the final request completion. It removes the +initial all-prefill interval and is decode-facing, but it can still contain +staggered prefill while later requests await their first token. +`Request decode tok/s` is the median per-request estimate +`(completion_tokens - 1) / (end - first_output)`; it assumes the first +observed streaming event accounts for one token. Neither metric is pure kernel +decode throughput. + +`Prompt tok/s to first` is the sum of server-reported prompt tokens divided by +the latest first-token arrival; it is a useful prefill-facing metric but still +includes admission, queueing, and transport. Report TTFT median/max alongside +all throughput metrics. + +The K8-vs-K1 comparison is the causal packing ablation. The K8-vs-llama +comparison is the product comparison. Five paired repeats, the exact command +and hashes recorded in each case, zero failures, and a fixed declared output +length are required before using results in a post. The standard prefill-facing +protocol uses 64 output tokens; the decode-heavy protocol above uses 256. +Variant gains are computed as the median of same-repeat ratios, not as a ratio +of independently aggregated medians. The summarizer rejects mismatched repeat +sets. It also marks whether each variant produced the same ordered output +hashes across at least two repeats; a one-repeat screen reports stability as +`n/a`, and an unstable result is a correctness warning, not a performance win. diff --git a/harness/benchmarks/concurrency/concurrent_benchmark.py b/harness/benchmarks/concurrency/concurrent_benchmark.py new file mode 100755 index 000000000..7a1e638e4 --- /dev/null +++ b/harness/benchmarks/concurrency/concurrent_benchmark.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Measure end-to-end output goodput and TTFT under concurrent streaming load.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import sys +import threading +import time +import urllib.request +from collections.abc import Iterable +from pathlib import Path +from typing import Any + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def load_prompts(path: Path) -> list[str]: + prompts = [] + for line_no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = raw.strip() + if not line: + continue + if line.startswith("{"): + value = json.loads(line).get("prompt") + if not isinstance(value, str) or not value: + raise ValueError(f"{path}:{line_no}: missing string 'prompt'") + prompts.append(value) + else: + prompts.append(line) + if not prompts: + raise ValueError(f"{path}: no prompts") + return prompts + + +def request_prompts(prompts: list[str], count: int, offset: int) -> list[str]: + if offset < 0: + raise ValueError("--prompt-offset must be >= 0") + if offset + count > len(prompts): + raise ValueError( + f"need prompts [{offset}, {offset + count}), but only " + f"{len(prompts)} were supplied; refusing to reuse prompts" + ) + return prompts[offset:offset + count] + + +def iter_sse_data(lines: Iterable[bytes]) -> Iterable[str]: + data: list[str] = [] + for raw in lines: + line = raw.decode("utf-8", errors="replace").rstrip("\r\n") + if not line: + if data: + yield "\n".join(data) + data.clear() + elif line.startswith("data:"): + data.append(line[5:].lstrip()) + if data: + yield "\n".join(data) + + +def stream_request(args: argparse.Namespace, prompt: str) -> dict[str, Any]: + started = time.perf_counter() + first = None + content: list[str] = [] + reasoning: list[str] = [] + completion_tokens = None + prompt_tokens = None + finish_reason = None + done_received = False + error = None + payload = { + "model": args.model, + "messages": [{"role": "user", "content": prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "max_tokens": args.max_tokens, + "temperature": args.temperature, + "seed": args.seed, + } + if args.ignore_eos: + payload["ignore_eos"] = True + headers = {"Content-Type": "application/json"} + if args.api_key: + headers["Authorization"] = f"Bearer {args.api_key}" + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=args.timeout) as response: + for data in iter_sse_data(response): + if data == "[DONE]": + done_received = True + break + event = json.loads(data) + usage = event.get("usage") or {} + if isinstance(usage.get("completion_tokens"), int): + completion_tokens = usage["completion_tokens"] + if isinstance(usage.get("prompt_tokens"), int): + prompt_tokens = usage["prompt_tokens"] + for choice in event.get("choices") or []: + if choice.get("finish_reason") is not None: + finish_reason = choice["finish_reason"] + delta = choice.get("delta") or {} + piece = delta.get("content") + thought = delta.get("reasoning_content") + if isinstance(piece, str) and piece: + first = first or time.perf_counter() + content.append(piece) + if isinstance(thought, str) and thought: + first = first or time.perf_counter() + reasoning.append(thought) + except Exception as exc: # preserve partial timing/output for diagnosis + error = f"{type(exc).__name__}: {exc}" + if error is None and not done_received: + error = "ProtocolError: stream ended before [DONE]" + elif error is None and finish_reason is None: + error = "ProtocolError: stream ended without a terminal finish_reason" + ended = time.perf_counter() + output = "".join(content) + reasoning_output = "".join(reasoning) + decode_duration = ended - first if first is not None and ended > first else None + request_decode_tok_s = ( + (completion_tokens - 1) / decode_duration + if isinstance(completion_tokens, int) and completion_tokens > 0 + and decode_duration is not None else None + ) + return { + "t_start": started, "t_first": first, "t_end": ended, + "duration_s": ended - started, + "ttft_s": first - started if first is not None else None, + "decode_duration_s": decode_duration, + "completion_tokens": completion_tokens, "prompt_tokens": prompt_tokens, + "finish_reason": finish_reason, "done_received": done_received, "error": error, + "content_sha256": sha256_text(output), + "reasoning_content_sha256": sha256_text(reasoning_output), + "content_chars": len(output), "reasoning_content_chars": len(reasoning_output), + "request_output_tok_s": ( + completion_tokens / (ended - started) + if completion_tokens is not None and ended > started else None + ), + "request_decode_tok_s": request_decode_tok_s, + } + + +def run_level( + clients: int, args: argparse.Namespace, prompts: list[str], offset: int, +) -> dict[str, Any]: + selected = request_prompts(prompts, clients, offset) + barrier = threading.Barrier(clients) + records: list[dict[str, Any] | None] = [None] * clients + + def worker(index: int) -> None: + barrier.wait() + record = stream_request(args, selected[index]) + record["prompt_index"] = offset + index + record["prompt_sha256"] = sha256_text(selected[index]) + records[index] = record + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(clients)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(args.timeout + 30) + completed = [record for record in records if record is not None] + hung = sum(thread.is_alive() for thread in threads) + failures = hung + sum(record["error"] is not None for record in completed) + ok = [record for record in completed if record["error"] is None] + starts = [record["t_start"] for record in completed] + ends = [record["t_end"] for record in completed] + level_start = min(starts) if starts else time.perf_counter() + wall = max(ends) - level_start if ends else 0.0 + for record in completed: + record["start_offset_s"] = record["t_start"] - level_start + + completion_counts = [r["completion_tokens"] for r in ok] + prompt_counts = [r["prompt_tokens"] for r in ok] + completion_complete = bool(ok) and all(isinstance(v, int) for v in completion_counts) + prompt_complete = bool(ok) and all(isinstance(v, int) for v in prompt_counts) + ttfts = [r["ttft_s"] for r in ok if r["ttft_s"] is not None] + first_window = ( + max(r["start_offset_s"] + r["ttft_s"] for r in ok) + if len(ttfts) == len(ok) and ok else None + ) + first_times = [r["t_first"] for r in ok if r["t_first"] is not None] + output_window = ( + max(r["t_end"] for r in ok) - min(first_times) + if len(first_times) == len(ok) and ok else None + ) + request_decode_rates = [ + r["request_decode_tok_s"] for r in ok + if r.get("request_decode_tok_s") is not None + ] + fixed_valid = ( + failures == 0 and len(ok) == clients + and completion_complete + and all(v == args.max_tokens for v in completion_counts) + ) if args.ignore_eos else None + prompt_hashes = [r["prompt_sha256"] for r in ok] + output_hashes = [ + [r["content_sha256"], r["reasoning_content_sha256"]] for r in ok + ] + digest = lambda value: sha256_text(json.dumps(value, separators=(",", ":"))) + return { + "clients": clients, "requests": clients, "requests_ok": len(ok), + "failures": failures, "wall_s": wall, + "start_skew_s": max(starts) - min(starts) if starts else None, + "completion_tokens_total": sum(completion_counts) if completion_complete else None, + "token_count_complete": completion_complete, + "fixed_token_workload_valid": fixed_valid, + "aggregate_tok_s": ( + sum(completion_counts) / wall if completion_complete and wall > 0 else None + ), + "aggregate_metric": "completion_tokens_per_level_wall_second", + "output_window_s": output_window, + "output_window_tok_s": ( + sum(completion_counts) / output_window + if completion_complete and output_window is not None and output_window > 0 + else None + ), + "output_window_metric": "completion_tokens_per_first_output_to_final_completion_second", + "request_decode_tok_s_median": ( + statistics.median(request_decode_rates) + if len(request_decode_rates) == len(ok) and ok else None + ), + "prompt_tokens_total": sum(prompt_counts) if prompt_complete else None, + "prompt_tokens_min": min(prompt_counts) if prompt_complete else None, + "prompt_tokens_max": max(prompt_counts) if prompt_complete else None, + "prompt_tokens_distinct": len(set(prompt_counts)) if prompt_complete else None, + "prompt_token_count_complete": prompt_complete, + "prompt_to_first_token_s": first_window, + "prompt_tokens_per_s_to_first_token": ( + sum(prompt_counts) / first_window + if prompt_complete and first_window is not None and first_window > 0 else None + ), + "ttft_median_s": statistics.median(ttfts) if ttfts else None, + "ttft_max_s": max(ttfts) if ttfts else None, + "selected_prompt_set_sha256": digest(prompt_hashes), + "selected_output_set_sha256": digest(output_hashes), + "requests_detail": completed, + } + + +def fmt(value: Any, spec: str = ".2f") -> str: + return format(value, spec) if isinstance(value, (int, float)) else "n/a" + + +def markdown(report: dict[str, Any]) -> str: + lines = [ + f"# Concurrent benchmark — {report['label']}", "", + "| C | Ok | Output goodput tok/s | Output-window tok/s | " + "Request decode tok/s | Prompt tok/s to first | Prompt range | " + "TTFT median s | TTFT max s | Wall s |", + "| ---: | ---: | ---: | ---: | ---: | ---: | :--- | ---: | ---: | ---: |", + ] + for level in report["levels"]: + lines.append( + f"| {level['clients']} | {level['requests_ok']}/{level['requests']} | " + f"{fmt(level['aggregate_tok_s'])} | " + f"{fmt(level['output_window_tok_s'])} | " + f"{fmt(level['request_decode_tok_s_median'])} | " + f"{fmt(level['prompt_tokens_per_s_to_first_token'])} | " + f"{fmt(level['prompt_tokens_min'], '.0f')}–{fmt(level['prompt_tokens_max'], '.0f')} | " + f"{fmt(level['ttft_median_s'], '.3f')} | {fmt(level['ttft_max_s'], '.3f')} | " + f"{fmt(level['wall_s'])} |" + ) + return "\n".join(lines) + "\n" + + +def level_failed(level: dict[str, Any], ignore_eos: bool) -> bool: + return bool( + level["failures"] + or not level["token_count_complete"] + or not level["prompt_token_count_complete"] + or (ignore_eos and level["fixed_token_workload_valid"] is not True) + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:18080/v1") + parser.add_argument("--api-key", default="") + parser.add_argument("--model", default="luce-dflash") + parser.add_argument("--clients", type=int, action="append", dest="client_levels") + parser.add_argument("--prompt-file", type=Path, required=True) + parser.add_argument("--prompt-offset", type=int, default=0) + parser.add_argument("--require-distinct-prompts", action="store_true", + help="Compatibility flag; this client always refuses reuse") + parser.add_argument("--max-tokens", type=int, default=64) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--ignore-eos", action="store_true") + parser.add_argument("--timeout", type=float, default=1200.0) + parser.add_argument("--cooldown", type=float, default=0.0) + parser.add_argument("--server-metadata-json", type=Path) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--label", default="") + return parser + + +def run(args: argparse.Namespace) -> int: + levels = args.client_levels or [1, 4, 8, 16] + if any(level < 1 for level in levels): + raise ValueError("--clients must be positive") + if args.prompt_offset < 0 or args.max_tokens < 1 or args.timeout <= 0: + raise ValueError("invalid offset, max-tokens, or timeout") + prompts = load_prompts(args.prompt_file) + results = [] + offset = args.prompt_offset + for index, clients in enumerate(levels): + if index and args.cooldown > 0: + time.sleep(args.cooldown) + print(f"[bench] C={clients} max_tokens={args.max_tokens}", flush=True) + results.append(run_level(clients, args, prompts, offset)) + offset += clients + metadata = ( + json.loads(args.server_metadata_json.read_text(encoding="utf-8")) + if args.server_metadata_json else {} + ) + report = { + "schema_version": 2, "label": args.label, "base_url": args.base_url, + "model": args.model, "max_tokens": args.max_tokens, + "temperature": args.temperature, "seed": args.seed, + "ignore_eos": args.ignore_eos, "prompt_offset": args.prompt_offset, + "prompt_file_sha256": hashlib.sha256(args.prompt_file.read_bytes()).hexdigest(), + "server_metadata": metadata, "levels": results, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(markdown(report), end="") + bad = any(level_failed(level, args.ignore_eos) for level in results) + return 1 if bad else 0 + + +def main() -> int: + try: + return run(build_parser().parse_args()) + except Exception as exc: + print(f"[bench] error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/generate_ragged_prompts.py b/harness/benchmarks/concurrency/generate_ragged_prompts.py new file mode 100755 index 000000000..e3dce8c49 --- /dev/null +++ b/harness/benchmarks/concurrency/generate_ragged_prompts.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Generate a small deterministic ragged-prompt manifest for concurrency runs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +PROFILES = { + "short": (250, 350, 450, 550), + "medium": (650, 850, 1150, 1350), + "long": (2000, 2600, 3400, 4000), +} + +WORD_BANK = ( + "systems engineers compare latency throughput scheduling memory kernels queues " + "batches requests tokens caches pages attention arithmetic bandwidth occupancy " + "profiling measurement fairness reproducibility workloads concurrency admission " + "prefill decoding evidence tradeoffs implementation validation production service" +).split() + + +def prompt_text(profile: str, cohort: str, index: int, target_words: int) -> str: + prefix = ( + f"Ragged benchmark {profile} cohort {cohort} request {index}. " + "Write a structured engineering analysis of the following observations, " + "including assumptions, likely bottlenecks, and a concise conclusion." + ).split() + words = list(prefix) + cursor = (index * 7 + target_words) % len(WORD_BANK) + while len(words) < target_words: + words.append(WORD_BANK[cursor % len(WORD_BANK)]) + cursor += 1 + return " ".join(words[:target_words]) + + +def build_records(profile: str) -> list[dict[str, object]]: + strata = PROFILES[profile] + layout = [ + ("c1", [sum(strata) // len(strata)]), + ("c4", list(strata)), + ("c8", list(strata) * 2), + ("c16", list(strata) * 4), + ] + records: list[dict[str, object]] = [] + for cohort, targets in layout: + for target in targets: + index = len(records) + records.append({ + "id": f"{profile}-{index:02d}", + "cohort": cohort, + "stratum": strata.index(target) if target in strata else "mean", + "target_words": target, + "prompt": prompt_text(profile, cohort, index, target), + }) + return records + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=sorted(PROFILES), required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.out.exists(): + parser.error(f"refusing to overwrite {args.out}") + args.out.parent.mkdir(parents=True, exist_ok=True) + records = build_records(args.profile) + args.out.write_text( + "".join(json.dumps(row, sort_keys=True) + "\n" for row in records), + encoding="utf-8", + ) + print(f"wrote {len(records)} prompts to {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/run_qwen36_concurrency.sh b/harness/benchmarks/concurrency/run_qwen36_concurrency.sh new file mode 100755 index 000000000..d809815bb --- /dev/null +++ b/harness/benchmarks/concurrency/run_qwen36_concurrency.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# Paired, fresh-process Qwen3.6 concurrency benchmark for Lucebox and llama.cpp. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO="${REPO:-$(cd -- "$SCRIPT_DIR/../../.." && pwd -P)}" +CLIENT="${CLIENT:-$SCRIPT_DIR/concurrent_benchmark.py}" +GENERATOR="${GENERATOR:-$SCRIPT_DIR/generate_ragged_prompts.py}" +SUMMARIZER="${SUMMARIZER:-$SCRIPT_DIR/summarize_concurrency.py}" + +MODEL="${MODEL:-}" +LUCE_SERVER_BIN="${LUCE_SERVER_BIN:-$REPO/server/build-hip/dflash_server}" +LLAMA_SERVER_BIN="${LLAMA_SERVER_BIN:-$(command -v llama-server 2>/dev/null || true)}" +OUT="${OUT:-$REPO/.harness-runs/qwen36-concurrency-$(date -u +%Y%m%dT%H%M%SZ)}" +REPEATS="${REPEATS:-1}" +WORKLOADS="${WORKLOADS:-short,medium,long}" +VARIANTS="${VARIANTS:-luce-k8,luce-k1,llama}" +CLIENTS="${CLIENTS:-1,4,8,16}" +PORT="${PORT:-18114}" +COOLDOWN_SECONDS="${COOLDOWN_SECONDS:-3}" +HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-600}" +MAX_TOKENS="${MAX_TOKENS:-64}" +WARMUP_TOKENS="${WARMUP_TOKENS:-8}" +SLOTS=16 + +usage() { + cat <<'EOF' +Usage: MODEL=/path/model.gguf [REPEATS=5] run_qwen36_concurrency.sh + +Runs fresh-server, same-concurrency warmup + measurement cases for luce-k8, +luce-k1, and llama at C=1/4/8/16. Defaults to one repeat for screening; use at +least five paired repeats for publication. For a decode-heavy comparison, set +WORKLOADS=short MAX_TOKENS=256 VARIANTS=luce-k8,llama. OUT must not already +exist. +EOF +} + +if [[ "${1:-}" == "--help" ]]; then usage; exit 0; fi +if [[ $# -ne 0 ]]; then usage >&2; exit 2; fi +for cmd in python3 curl sha256sum; do command -v "$cmd" >/dev/null || { echo "missing $cmd" >&2; exit 2; }; done +[[ -r "$MODEL" ]] || { echo "set MODEL to a readable GGUF" >&2; exit 2; } +[[ -x "$LUCE_SERVER_BIN" ]] || { echo "missing Lucebox server: $LUCE_SERVER_BIN" >&2; exit 2; } +[[ -x "$LLAMA_SERVER_BIN" ]] || { echo "missing llama.cpp server: $LLAMA_SERVER_BIN" >&2; exit 2; } +[[ "$REPEATS" =~ ^[1-9][0-9]*$ ]] || { echo "REPEATS must be positive" >&2; exit 2; } +[[ ! -e "$OUT" ]] || { echo "refusing to overwrite $OUT" >&2; exit 2; } +ambient_tuning="$(env | grep -E '^(GGML_|DFLASH_|LUCE_|HIP_|ROCR_|HSA_|LD_PRELOAD=|LD_LIBRARY_PATH=)' \ + | grep -v '^LUCE_SERVER_BIN=' || true)" +if [[ -n "$ambient_tuning" ]]; then + echo "refusing ambient GPU/backend tuning variables:" >&2 + echo "$ambient_tuning" >&2 + exit 2 +fi +MODEL_SHA256="$(sha256sum "$MODEL" | awk '{print $1}')" + +IFS=, read -r -a workload_list <<< "$WORKLOADS" +IFS=, read -r -a variant_list <<< "$VARIANTS" +IFS=, read -r -a client_list <<< "$CLIENTS" +declare -A prompt_offsets=([1]=0 [4]=1 [8]=5 [16]=13) +for c in "${client_list[@]}"; do + [[ -n "${prompt_offsets[$c]+yes}" ]] || { echo "supported CLIENTS are 1,4,8,16" >&2; exit 2; } +done +for v in "${variant_list[@]}"; do + [[ "$v" == luce-k8 || "$v" == luce-k1 || "$v" == llama ]] || { echo "unknown variant $v" >&2; exit 2; } +done + +mkdir -p "$OUT/prompts" +for workload in "${workload_list[@]}"; do + python3 "$GENERATOR" --profile "$workload" --out "$OUT/prompts/$workload.jsonl" +done + +server_pid="" +stop_server() { + if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then + kill "$server_pid" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "$server_pid" 2>/dev/null || break + sleep 1 + done + kill -9 "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi + server_pid="" +} +trap stop_server EXIT INT TERM + +wait_health() { + local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) + while (( SECONDS < deadline )); do + kill -0 "$server_pid" 2>/dev/null || return 1 + curl -fsS --max-time 2 "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +write_metadata() { + local path="$1" variant="$2" workload="$3" clients="$4" repeat="$5" binary="$6" max_prefills="$7" command_file="$8" + python3 -c 'import hashlib,json,pathlib,subprocess,sys +p,variant,workload,clients,repeat,binary,max_prefills,cmd_file,model_sha,prompts,repo=sys.argv[1:] +digest=lambda x: hashlib.sha256(pathlib.Path(x).read_bytes()).hexdigest() +libs={} +for line in subprocess.run(["ldd",binary],text=True,capture_output=True).stdout.splitlines(): + fields=line.replace("=>"," ").split() + paths=[x for x in fields if x.startswith("/") and pathlib.Path(x).is_file()] + for lib in paths: libs[str(pathlib.Path(lib).resolve())]=digest(lib) +lucebox_git_head=subprocess.run(["git","-C",repo,"rev-parse","HEAD"],text=True,capture_output=True).stdout.strip() or None +server_version=None +if variant == "llama": + version=subprocess.run([binary,"--version"],text=True,capture_output=True,timeout=30) + server_version="\n".join(x.strip() for x in (version.stdout,version.stderr) if x.strip()) or None + if version.returncode != 0 or server_version is None: + raise RuntimeError(f"cannot identify llama.cpp source version from {binary} --version") +obj={"variant":variant,"workload":workload,"clients":int(clients),"repeat":int(repeat), + "max_concurrent_prefills":int(max_prefills),"server_binary":str(pathlib.Path(binary).resolve()), + "server_binary_sha256":digest(binary),"model_sha256":model_sha, + "prompt_file_sha256":digest(prompts),"server_command":pathlib.Path(cmd_file).read_text().strip(), + "resolved_shared_library_sha256":libs, + "lucebox_git_head":lucebox_git_head if variant != "llama" else None, + "server_version":server_version} +pathlib.Path(p).write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")' \ + "$path" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" "$command_file" "$MODEL_SHA256" "$OUT/prompts/$workload.jsonl" "$REPO" +} + +run_case() { + local repeat="$1" workload="$2" clients="$3" variant="$4" + local max_ctx timeout capacity max_prefills binary model_id + if [[ "$workload" == long ]]; then + max_ctx=8192; timeout=1800 + else + max_ctx=4096; timeout=1200 + fi + capacity=$((SLOTS * max_ctx)) + local case_dir="$OUT/$workload/c$clients/r$repeat/$variant" + mkdir -p "$case_dir" + local -a command launch_command + if [[ "$variant" == llama ]]; then + binary="$LLAMA_SERVER_BIN"; model_id=qwen36-llama; max_prefills=0 + command=("$binary" -m "$MODEL" -ngl all --parallel "$SLOTS" -c "$capacity" + -b 2048 -ub 512 --cont-batching --no-context-shift --no-mmap -fa on + -ctk q4_0 -ctv q4_0 --no-cache-prompt --host 127.0.0.1 --port "$PORT" --alias "$model_id") + else + binary="$LUCE_SERVER_BIN"; model_id=qwen36-luce + [[ "$variant" == luce-k8 ]] && max_prefills=8 || max_prefills=1 + command=("$binary" "$MODEL" --target-device hip:0 --paged-attention + --max-concurrency "$SLOTS" --kv-pool-tokens "$capacity" --max-ctx "$max_ctx" + --cache-type-k q4_0 --cache-type-v q4_0 --fa-window 0 + --prefix-cache-slots 0 --prefill-cache-slots 0 --admission-coalesce-ms 5 + --host 127.0.0.1 --port "$PORT" --model-name "$model_id") + fi + if [[ "$variant" == llama ]]; then + launch_command=("${command[@]}") + else + launch_command=(env DFLASH_IGNORE_EOS=1 + DFLASH_MIN_TOKENS="$WARMUP_TOKENS" + DFLASH_MAX_CONCURRENT_PREFILLS="$max_prefills" "${command[@]}") + fi + printf '%q ' "${launch_command[@]}" > "$case_dir/server-command.txt"; printf '\n' >> "$case_dir/server-command.txt" + write_metadata "$case_dir/server-metadata.json" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" "$case_dir/server-command.txt" + + echo "[run] $workload C=$clients repeat=$repeat variant=$variant" + "${launch_command[@]}" > "$case_dir/server.log" 2>&1 & + server_pid=$! + if ! wait_health; then + tail -n 80 "$case_dir/server.log" >&2 || true + stop_server + sleep "$COOLDOWN_SECONDS" + return 1 + fi + + local offset="${prompt_offsets[$clients]}" prompts="$OUT/prompts/$workload.jsonl" + local status=0 + if ! python3 "$CLIENT" --base-url "http://127.0.0.1:$PORT/v1" --model "$model_id" \ + --clients "$clients" --prompt-file "$prompts" --prompt-offset "$offset" \ + --require-distinct-prompts --max-tokens "$WARMUP_TOKENS" --temperature 0 \ + --ignore-eos --timeout "$timeout" --cooldown 0 --out "$case_dir/warmup.json" \ + --label "$variant $workload C=$clients warmup" > "$case_dir/warmup.txt"; then + status=1 + fi + if (( status == 0 )) && ! python3 "$CLIENT" --base-url "http://127.0.0.1:$PORT/v1" --model "$model_id" \ + --clients "$clients" --prompt-file "$prompts" --prompt-offset "$offset" \ + --require-distinct-prompts --max-tokens "$MAX_TOKENS" --temperature 0 \ + --ignore-eos --timeout "$timeout" --cooldown 0 \ + --server-metadata-json "$case_dir/server-metadata.json" --out "$case_dir/bench.json" \ + --label "$variant $workload C=$clients repeat=$repeat" | tee "$case_dir/bench.txt"; then + status=1 + fi + stop_server + sleep "$COOLDOWN_SECONDS" + return "$status" +} + +case_failures=0 +for ((repeat=1; repeat<=REPEATS; repeat++)); do + for workload in "${workload_list[@]}"; do + for c_index in "${!client_list[@]}"; do + clients="${client_list[$c_index]}" + # Rotate start variant by case so one engine is not always hot or cold. + shift_by=$(((repeat + c_index) % ${#variant_list[@]})) + for ((i=0; i<${#variant_list[@]}; i++)); do + variant="${variant_list[$(((i + shift_by) % ${#variant_list[@]}))]}" + if ! run_case "$repeat" "$workload" "$clients" "$variant"; then + echo "[run] failed: $workload C=$clients repeat=$repeat variant=$variant" >&2 + case_failures=$((case_failures + 1)) + fi + done + done + done +done + +summary_status=0 +python3 "$SUMMARIZER" "$OUT" --out "$OUT/summary.md" || summary_status=$? +if (( case_failures > 0 || summary_status != 0 )); then + echo "[run] completed with $case_failures failed case(s)" >&2 + exit 1 +fi +echo "[run] complete: $OUT" diff --git a/harness/benchmarks/concurrency/summarize_concurrency.py b/harness/benchmarks/concurrency/summarize_concurrency.py new file mode 100755 index 000000000..69563a35c --- /dev/null +++ b/harness/benchmarks/concurrency/summarize_concurrency.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Summarize paired Lucebox/llama.cpp concurrency benchmark reports.""" + +from __future__ import annotations + +import argparse +import json +import statistics +from collections import defaultdict +from pathlib import Path + + +def load_reports(root: Path) -> list[dict]: + reports = [] + for path in sorted(root.rglob("bench.json")): + report = json.loads(path.read_text(encoding="utf-8")) + meta = report.get("server_metadata") or {} + if len(report.get("levels", [])) != 1: + raise ValueError(f"{path}: expected exactly one client level") + level = report["levels"][0] + if ( + level.get("failures") + or not level.get("token_count_complete") + or not level.get("prompt_token_count_complete") + ): + raise ValueError(f"{path}: failed or incomplete token accounting") + if report.get("ignore_eos") is not True: + raise ValueError(f"{path}: fixed-token protocol is required") + if level.get("fixed_token_workload_valid") is not True: + raise ValueError(f"{path}: fixed-token validation failed") + reports.append({"path": path, "report": report, "level": level, "meta": meta}) + if not reports: + raise ValueError(f"{root}: no bench.json files found") + return reports + + +def median(values: list[float]) -> float: + return statistics.median(values) + + +def paired_delta( + grouped: dict[tuple[str, int, str], list[dict]], + workload: str, + clients: int, + variant: str, + items: list[dict], + prompt_hashes: set[str], + other: str, + metric: str, +) -> str: + peers = grouped.get((workload, clients, other), []) + if not peers: + return "n/a" + output_hashes = { + item["level"].get("selected_output_set_sha256") for item in items + } + peer_output_hashes = { + item["level"].get("selected_output_set_sha256") for item in peers + } + if len(output_hashes) > 1 or len(peer_output_hashes) > 1: + return "n/a" + peer_hashes = {p["level"]["selected_prompt_set_sha256"] for p in peers} + if peer_hashes != prompt_hashes: + raise ValueError(f"{workload} C={clients}: {variant}/{other} prompts differ") + by_repeat = {int(item["meta"]["repeat"]): item for item in items} + peers_by_repeat = {int(item["meta"]["repeat"]): item for item in peers} + if by_repeat.keys() != peers_by_repeat.keys(): + raise ValueError( + f"{workload} C={clients}: {variant}/{other} repeat sets differ" + ) + ratios = [] + for repeat in sorted(by_repeat): + value = by_repeat[repeat]["level"].get(metric) + base = peers_by_repeat[repeat]["level"].get(metric) + if value is None or base is None: + return "n/a" + if base <= 0: + raise ValueError( + f"{workload} C={clients} repeat={repeat}: " + f"non-positive {other} {metric}" + ) + ratios.append(value / base - 1.0) + return f"{median(ratios) * 100:+.1f}%" + + +def summarize(reports: list[dict]) -> str: + grouped: dict[tuple[str, int, str], list[dict]] = defaultdict(list) + for item in reports: + meta, level = item["meta"], item["level"] + key = (str(meta["workload"]), int(level["clients"]), str(meta["variant"])) + grouped[key].append(item) + for key, items in grouped.items(): + repeats = [int(item["meta"]["repeat"]) for item in items] + if len(repeats) != len(set(repeats)): + raise ValueError(f"{key}: duplicate repeat") + + lines = [ + "# Concurrency benchmark summary", "", + "Aggregate output goodput includes queueing, prefill, and decode. " + "Output-window goodput starts at the first observed output and is decode-facing, " + "but it can include staggered prefill. Prompt tok/s to first token includes " + "admission and TTFT.", "", + "| Workload | C | Variant | Repeats | Output goodput tok/s | " + "Output-window tok/s | Request decode tok/s | Prompt tok/s to first | " + "TTFT median s | TTFT max s | Stable output | vs llama | Decode vs llama | K8 vs K1 |", + "| :--- | ---: | :--- | ---: | ---: | ---: | ---: | ---: | ---: | " + "---: | :---: | ---: | ---: | ---: |", + ] + for workload, clients, variant in sorted(grouped): + items = grouped[(workload, clients, variant)] + hashes = {item["level"]["selected_prompt_set_sha256"] for item in items} + if len(hashes) != 1: + raise ValueError(f"{workload} C={clients} {variant}: prompt sets differ") + goodput = median([item["level"]["aggregate_tok_s"] for item in items]) + output_window_values = [ + item["level"].get("output_window_tok_s") for item in items + if item["level"].get("output_window_tok_s") is not None + ] + output_window = median(output_window_values) if output_window_values else None + request_decode_values = [ + item["level"].get("request_decode_tok_s_median") for item in items + if item["level"].get("request_decode_tok_s_median") is not None + ] + request_decode = median(request_decode_values) if request_decode_values else None + prompt_rate_values = [ + item["level"]["prompt_tokens_per_s_to_first_token"] for item in items + if item["level"].get("prompt_tokens_per_s_to_first_token") is not None + ] + prompt_rate = median(prompt_rate_values) if prompt_rate_values else None + ttft_median = median([item["level"]["ttft_median_s"] for item in items]) + ttft_max = median([item["level"]["ttft_max_s"] for item in items]) + output_hashes = { + item["level"].get("selected_output_set_sha256") for item in items + } + stable = ( + "n/a" if len(items) < 2 + else "yes" if len(output_hashes) == 1 + else "NO" + ) + + vs_llama = ( + paired_delta( + grouped, workload, clients, variant, items, hashes, + "llama", "aggregate_tok_s", + ) + if variant == "luce-k8" else "—" + ) + decode_vs_llama = ( + paired_delta( + grouped, workload, clients, variant, items, hashes, + "llama", "output_window_tok_s", + ) + if variant == "luce-k8" else "—" + ) + vs_k1 = ( + paired_delta( + grouped, workload, clients, variant, items, hashes, + "luce-k1", "aggregate_tok_s", + ) + if variant == "luce-k8" else "—" + ) + output_window_text = f"{output_window:.2f}" if output_window is not None else "n/a" + request_decode_text = f"{request_decode:.2f}" if request_decode is not None else "n/a" + prompt_rate_text = f"{prompt_rate:.2f}" if prompt_rate is not None else "n/a" + lines.append( + f"| {workload} | {clients} | {variant} | {len(items)} | {goodput:.2f} | " + f"{output_window_text} | {request_decode_text} | {prompt_rate_text} | " + f"{ttft_median:.3f} | {ttft_max:.3f} | {stable} | {vs_llama} | " + f"{decode_vs_llama} | {vs_k1} |" + ) + lines.append("") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("--out", type=Path) + args = parser.parse_args() + text = summarize(load_reports(args.root)) + if args.out: + args.out.write_text(text + "\n", encoding="utf-8") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/test_concurrency_tools.py b/harness/benchmarks/concurrency/test_concurrency_tools.py new file mode 100644 index 000000000..925694d29 --- /dev/null +++ b/harness/benchmarks/concurrency/test_concurrency_tools.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Unit tests for the deterministic prompt generator and compact summarizer.""" + +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).parent + + +def load(name: str): + path = HERE / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +generator = load("generate_ragged_prompts") +summarizer = load("summarize_concurrency") + + +class PromptGeneratorTests(unittest.TestCase): + def test_cohorts_are_disjoint_ragged_and_mean_matched(self) -> None: + records = generator.build_records("short") + self.assertEqual(len(records), 29) + self.assertEqual( + [row["cohort"] for row in records], + ["c1"] + ["c4"] * 4 + ["c8"] * 8 + ["c16"] * 16, + ) + self.assertEqual(len({row["prompt"] for row in records}), 29) + by_cohort = { + cohort: [row for row in records if row["cohort"] == cohort] + for cohort in ("c1", "c4", "c8", "c16") + } + means = { + cohort: sum(row["target_words"] for row in rows) / len(rows) + for cohort, rows in by_cohort.items() + } + self.assertEqual(len(set(means.values())), 1) + for cohort in ("c4", "c8", "c16"): + self.assertEqual(len({row["target_words"] for row in by_cohort[cohort]}), 4) + for row in records: + self.assertEqual(len(row["prompt"].split()), row["target_words"]) + + +class SummarizerTests(unittest.TestCase): + @staticmethod + def item( + variant: str, + goodput: float, + output_window: float | None = None, + *, + repeat: int = 1, + output_hash: str = "same-outputs", + ) -> dict: + return { + "meta": {"workload": "short", "variant": variant, "repeat": repeat}, + "level": { + "clients": 8, + "aggregate_tok_s": goodput, + "output_window_tok_s": output_window if output_window is not None else goodput, + "request_decode_tok_s_median": goodput / 8, + "prompt_tokens_per_s_to_first_token": 100.0, + "ttft_median_s": 1.0, + "ttft_max_s": 2.0, + "selected_prompt_set_sha256": "same-prompts", + "selected_output_set_sha256": output_hash, + }, + } + + def test_summary_reports_product_and_packing_deltas(self) -> None: + text = summarizer.summarize([ + self.item("luce-k8", 20.0), + self.item("luce-k1", 10.0), + self.item("llama", 8.0), + ]) + self.assertIn("+150.0%", text) + self.assertIn("+100.0%", text) + self.assertIn("Decode vs llama", text) + + def test_summary_uses_same_repeat_ratios(self) -> None: + reports = [] + for repeat, luce, llama in ( + (1, 10.0, 1.0), + (2, 20.0, 90.0), + (3, 100.0, 50.0), + ): + reports.extend([ + self.item("luce-k8", luce, repeat=repeat), + self.item("llama", llama, repeat=repeat), + ]) + text = summarizer.summarize(reports) + luce_row = next(line for line in text.splitlines() if "| luce-k8 |" in line) + self.assertIn("+100.0%", luce_row) + self.assertNotIn("-60.0%", luce_row) + + def test_summary_rejects_mismatched_repeat_sets(self) -> None: + reports = [ + self.item("luce-k8", 20.0, repeat=1), + self.item("luce-k8", 22.0, repeat=2), + self.item("llama", 10.0, repeat=1), + ] + with self.assertRaisesRegex(ValueError, "repeat sets differ"): + summarizer.summarize(reports) + + def test_single_repeat_does_not_claim_stability(self) -> None: + text = summarizer.summarize([self.item("llama", 8.0)]) + row = next(line for line in text.splitlines() if "| llama |" in line) + self.assertEqual(row.split("|")[11].strip(), "n/a") + + def test_multiple_repeats_report_output_stability(self) -> None: + stable = summarizer.summarize([ + self.item("llama", 8.0, repeat=1), + self.item("llama", 9.0, repeat=2), + ]) + stable_row = next(line for line in stable.splitlines() if "| llama |" in line) + self.assertEqual(stable_row.split("|")[11].strip(), "yes") + + unstable = summarizer.summarize([ + self.item("llama", 8.0, repeat=1, output_hash="first"), + self.item("llama", 9.0, repeat=2, output_hash="second"), + ]) + unstable_row = next(line for line in unstable.splitlines() if "| llama |" in line) + self.assertEqual(unstable_row.split("|")[11].strip(), "NO") + + def test_unstable_output_suppresses_comparison_deltas(self) -> None: + reports = [ + self.item("luce-k8", 20.0, repeat=1, output_hash="luce-a"), + self.item("luce-k8", 22.0, repeat=2, output_hash="luce-b"), + self.item("llama", 10.0, repeat=1, output_hash="llama"), + self.item("llama", 11.0, repeat=2, output_hash="llama"), + ] + row = next( + line for line in summarizer.summarize(reports).splitlines() + if "| luce-k8 |" in line + ) + self.assertEqual(row.split("|")[12].strip(), "n/a") + self.assertEqual(row.split("|")[13].strip(), "n/a") + + peer_unstable = [ + self.item("luce-k8", 20.0, repeat=1, output_hash="luce"), + self.item("luce-k8", 22.0, repeat=2, output_hash="luce"), + self.item("llama", 10.0, repeat=1, output_hash="llama-a"), + self.item("llama", 11.0, repeat=2, output_hash="llama-b"), + ] + peer_row = next( + line for line in summarizer.summarize(peer_unstable).splitlines() + if "| luce-k8 |" in line + ) + self.assertEqual(peer_row.split("|")[12].strip(), "n/a") + self.assertEqual(peer_row.split("|")[13].strip(), "n/a") + + def test_summary_reports_ttft_median_and_max(self) -> None: + text = summarizer.summarize([self.item("llama", 8.0)]) + self.assertIn("TTFT median s | TTFT max s", text) + row = next(line for line in text.splitlines() if "| llama |" in line) + self.assertEqual(row.split("|")[9].strip(), "1.000") + self.assertEqual(row.split("|")[10].strip(), "2.000") + + def test_load_reports_rejects_missing_prompt_usage(self) -> None: + report = { + "ignore_eos": True, + "server_metadata": {"workload": "short", "variant": "llama", "repeat": 1}, + "levels": [{ + "failures": 0, + "token_count_complete": True, + "prompt_token_count_complete": False, + "fixed_token_workload_valid": True, + }], + } + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "bench.json" + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "incomplete token accounting"): + summarizer.load_reports(Path(root)) + + def test_load_reports_requires_fixed_token_protocol(self) -> None: + report = { + "ignore_eos": False, + "server_metadata": {"workload": "short", "variant": "llama", "repeat": 1}, + "levels": [{ + "failures": 0, + "token_count_complete": True, + "prompt_token_count_complete": True, + "fixed_token_workload_valid": None, + }], + } + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "bench.json" + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "fixed-token protocol"): + summarizer.load_reports(Path(root)) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/benchmarks/concurrency/test_concurrent_benchmark.py b/harness/benchmarks/concurrency/test_concurrent_benchmark.py new file mode 100644 index 000000000..201f3464f --- /dev/null +++ b/harness/benchmarks/concurrency/test_concurrent_benchmark.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Focused tests for concurrent_benchmark.py.""" + +from __future__ import annotations + +import argparse +import importlib.util +import time +import unittest +from pathlib import Path +from unittest import mock + +SCRIPT = Path(__file__).with_name("concurrent_benchmark.py") +SPEC = importlib.util.spec_from_file_location("concurrent_benchmark", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +benchmark = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(benchmark) + + +class BenchmarkTests(unittest.TestCase): + def test_sse_parser_handles_events_and_done(self) -> None: + lines = [ + b'data: {"choices":[{"delta":{"content":"hi"}}]}\n', b"\n", + b"data: [DONE]\n", b"\n", + ] + self.assertEqual( + list(benchmark.iter_sse_data(lines)), + ['{"choices":[{"delta":{"content":"hi"}}]}', "[DONE]"], + ) + + def test_prompt_selection_never_wraps(self) -> None: + self.assertEqual(benchmark.request_prompts(["a", "b", "c"], 2, 1), ["b", "c"]) + with self.assertRaisesRegex(ValueError, "refusing to reuse"): + benchmark.request_prompts(["a", "b"], 2, 1) + + def test_level_uses_exact_usage_and_first_token_window(self) -> None: + prompt_counts = iter((10, 30)) + + def fake_request(_args: argparse.Namespace, prompt: str) -> dict: + start = time.perf_counter() + return { + "t_start": start, "t_first": start + 0.5, "t_end": start + 1.0, + "duration_s": 1.0, "ttft_s": 0.5, "decode_duration_s": 0.5, + "completion_tokens": 8, "prompt_tokens": next(prompt_counts), + "finish_reason": "length", "error": None, + "content_sha256": benchmark.sha256_text(prompt + " output"), + "reasoning_content_sha256": benchmark.sha256_text(""), + "content_chars": 6, "reasoning_content_chars": 0, + "request_output_tok_s": 8.0, "request_decode_tok_s": 14.0, + } + + args = argparse.Namespace(max_tokens=8, ignore_eos=True, timeout=2.0) + with mock.patch.object(benchmark, "stream_request", side_effect=fake_request): + level = benchmark.run_level(2, args, ["first", "second"], 0) + self.assertEqual(level["completion_tokens_total"], 16) + self.assertEqual(level["prompt_tokens_total"], 40) + self.assertTrue(level["fixed_token_workload_valid"]) + self.assertAlmostEqual( + level["output_window_tok_s"], + 16 / level["output_window_s"], + ) + self.assertEqual(level["request_decode_tok_s_median"], 14.0) + self.assertAlmostEqual( + level["prompt_tokens_per_s_to_first_token"], + 40 / level["prompt_to_first_token_s"], + ) + + def test_stream_request_keeps_usage_separate_from_sse_chunks(self) -> None: + class Response: + def __enter__(self): return self + def __exit__(self, *_args): return None + def __iter__(self): + return iter([ + b'data: {"choices":[{"delta":{"content":"one chunk"}}]}\n', b"\n", + b'data: {"choices":[{"delta":{},"finish_reason":"length"}]}\n', b"\n", + b'data: {"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":64}}\n', b"\n", + b"data: [DONE]\n", b"\n", + ]) + + args = argparse.Namespace( + model="m", max_tokens=64, temperature=0.0, seed=1, ignore_eos=True, + api_key="", base_url="http://localhost/v1", timeout=2.0, + ) + with mock.patch.object(benchmark.urllib.request, "urlopen", return_value=Response()): + record = benchmark.stream_request(args, "prompt") + self.assertEqual(record["completion_tokens"], 64) + self.assertEqual(record["prompt_tokens"], 12) + self.assertTrue(record["done_received"]) + self.assertIsNone(record["error"]) + self.assertIsNotNone(record["request_decode_tok_s"]) + self.assertEqual(record["content_sha256"], benchmark.sha256_text("one chunk")) + + def test_stream_request_rejects_clean_eof_without_done(self) -> None: + class Response: + def __enter__(self): return self + def __exit__(self, *_args): return None + def __iter__(self): + return iter([ + b'data: {"choices":[{"delta":{"content":"partial"}}]}\n', b"\n", + b'data: {"choices":[{"delta":{},"finish_reason":"length"}]}\n', b"\n", + b'data: {"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":64}}\n', b"\n", + ]) + + args = argparse.Namespace( + model="m", max_tokens=64, temperature=0.0, seed=1, ignore_eos=True, + api_key="", base_url="http://localhost/v1", timeout=2.0, + ) + with mock.patch.object(benchmark.urllib.request, "urlopen", return_value=Response()): + record = benchmark.stream_request(args, "prompt") + self.assertFalse(record["done_received"]) + self.assertIn("before [DONE]", record["error"]) + + def test_missing_prompt_usage_fails_level(self) -> None: + level = { + "failures": 0, + "token_count_complete": True, + "prompt_token_count_complete": False, + "fixed_token_workload_valid": True, + } + self.assertTrue(benchmark.level_failed(level, ignore_eos=True)) + + +if __name__ == "__main__": + unittest.main() From ad3e5c2de6224501d2fa434fa519cc8c0b717215 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Fri, 14 Aug 2026 07:58:21 +0000 Subject: [PATCH 2/3] bench(concurrency): add canonical workload matrix --- harness/benchmarks/README.md | 3 +- harness/benchmarks/concurrency/README.md | 120 +++++++++ .../concurrency/attach_ddtree_metrics.py | 97 ++++++++ .../canonical_concurrent_benchmark.py | 235 ++++++++++++++++++ .../concurrency/concurrent_benchmark.py | 31 ++- .../concurrency/generate_blog_prompts.py | 34 +++ .../concurrency/raw_prompt_identity.jinja | 1 + .../run_qwen36_canonical_concurrency.sh | 195 +++++++++++++++ .../summarize_canonical_concurrency.py | 139 +++++++++++ .../concurrency/test_attach_ddtree_metrics.py | 59 +++++ .../test_canonical_concurrent_benchmark.py | 231 +++++++++++++++++ .../concurrency/test_concurrent_benchmark.py | 32 +++ 12 files changed, 1173 insertions(+), 4 deletions(-) create mode 100755 harness/benchmarks/concurrency/attach_ddtree_metrics.py create mode 100755 harness/benchmarks/concurrency/canonical_concurrent_benchmark.py create mode 100755 harness/benchmarks/concurrency/generate_blog_prompts.py create mode 100644 harness/benchmarks/concurrency/raw_prompt_identity.jinja create mode 100755 harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh create mode 100755 harness/benchmarks/concurrency/summarize_canonical_concurrency.py create mode 100644 harness/benchmarks/concurrency/test_attach_ddtree_metrics.py create mode 100644 harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py diff --git a/harness/benchmarks/README.md b/harness/benchmarks/README.md index 722e875d9..45bf2e321 100644 --- a/harness/benchmarks/README.md +++ b/harness/benchmarks/README.md @@ -4,7 +4,8 @@ These checks are separate from the client harness launchers. They compare Lucebo generation against a llama.cpp baseline on the same target GGUF, using small deterministic prompts. -For the paired ragged C1/C4/C8/C16 serving benchmark, see +For the paired ragged C1/C4/C8/C16 serving benchmark and the concurrent +HumanEval/GSM8K/Math500/agent suite runner, see [`concurrency/`](concurrency/README.md). Use this when you want to know whether a server change affects output quality or diff --git a/harness/benchmarks/concurrency/README.md b/harness/benchmarks/concurrency/README.md index c310251f5..9f2adc7b1 100644 --- a/harness/benchmarks/concurrency/README.md +++ b/harness/benchmarks/concurrency/README.md @@ -5,6 +5,126 @@ prefill and concurrent decode. It is intentionally small: one streaming client, one fresh-process runner, one deterministic prompt generator, and one summary script. +## Canonical and blog workloads + +The synthetic ragged profiles below isolate serving mechanics. To measure how +draft acceptance changes with real workload structure, use the canonical-suite +runner: + +```bash +MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +REPEATS=3 \ +harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh +``` + +It runs paged concurrent AR by default. Set `VARIANTS=blog-ddtree` and provide +`DRAFT_MODEL` to opt into the Strix Halo decode recipe from the AMD post: +the Q8_0 Qwen3.6 drafter, +`DFLASH27B_DRAFT_SWA=2048`, `--ddtree-budget 22`, `--fast-rollback`, and 128 +forced output tokens. Adaptive fallback is disabled so every eligible decode +step measures the speculative path. It intentionally keeps paged attention on, +including at C=1, because the goal is to compare the concurrent implementation +with the standalone blog result rather than silently route C=1 elsewhere. +The serving-only differences are stated rather than hidden: every level uses +the same 16-slot concurrent graph/pool and Q4_0 paged target KV, matching the +established concurrency configuration. The standalone `test_dflash` blog bench +has neither a paged multi-sequence pool nor idle serving slots. +The DDTree variant requires a concurrent-serving build that emits +`[concurrency-metrics]` records; the harness does not substitute AR telemetry. +The speculative row also fails closed unless every measured response has a +matching `[concurrency-metrics]` record with positive DDTree work. Its JSON +report stores DDTree steps, accepted children, target forwards, mean accepted +length, and acceptance rate (the blog's 16 draft candidates per step). +An independently selectable `adaptive-ddtree` variant uses the same blog setup +with the runtime's adaptive fallback enabled; this measures whether one +configuration can retain high-acceptance workloads and fall back on low-yield +ones. + +The workloads are: + +- `he-raw`: the exact ten raw prompts imported from `server/scripts/bench_he.py`; + an identity chat template preserves byte-for-byte blog prompt input. +- `he`, `gsm`, `math`, and `agent`: the checked-in canonical JSONL suites under + `harness/benchmarks/prompts`, preserving all system/user message roles. + +Each concurrency level processes the whole suite in fixed-width waves. The +ten-case suites use C=1/2/5/10; the six-case agent suite uses C=1/2/3/6. The +runner rejects a level that would create a smaller tail wave. This makes every +C comparison use the same prompt set without duplicating a prompt inside a run. +The C=1 case still launches exactly one request; the other 15 slots stay idle. +Use `SUITES=he-raw,gsm`, `VARIANTS=blog-ddtree`, or `CLIENTS=1,5` for a smaller +screen. +For a C=3 crossover measurement on a ten-case suite, set +`SUITES=he-raw CLIENTS=3 CASE_LIMIT=9`. This selects the first nine cases and runs three full +C=3 waves; the report records the limit. It never duplicates the tenth prompt +or mislabels a one-request tail as C=3. +Before starting the next wave, the client waits for a matching scheduler +retirement marker for every response. The wait is included in output goodput +wall time. This prevents nominal C=1 waves from accumulating as hidden C>1 work +when the HTTP stream closes slightly before slot retirement. + +The raw HumanEval row is the direct blog-parity check. The chat-form HumanEval, +GSM8K, Math500, and agent rows answer the separate product question: whether +the drafter continues to pay off after the server's normal chat template and +on less code-like generations. Do not pool acceptance or throughput across +these suites. + +### Strix Halo screening guidance + +One 2026-08-14 screening repeat on gfx1151 used the exact raw HumanEval prompts, +128 forced output tokens, the Q8_0/SWA=2048 blog drafter, budget 22, and the +fixed 16-slot Q4_0 paged serving configuration. C=3 uses the first nine prompts; +C=4 uses the first eight, so both are composed entirely of full waves. + +| C | Cases | AR goodput tok/s | DDTree goodput tok/s | DDTree vs AR | DDTree AL | Acceptance | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 10 | 11.59 | 15.17 | +30.9% | 5.40 | 33.8% | +| 2 | 10 | 16.66 | 22.32 | +34.0% | 5.60 | 35.0% | +| 3 | 9 | 18.88 | 25.47 | +34.9% | 6.11 | 38.2% | +| 4 | 8 | 31.43 | 28.44 | -9.5% | 5.83 | 36.4% | +| 5 | 10 | 37.50 | 25.05 | -33.2% | 5.59 | 34.9% | +| 10 | 10 | 56.13 | 24.48 | -56.4% | 5.59 | 34.9% | + +The reasoning suites were then run for five clean repeats with 96 forced output +tokens. C=3 uses nine cases and C=4 uses eight, so every repeat contains only +full-width waves. `DDTree vs AR` is the median of the five same-repeat ratios, +not a ratio of independently rounded medians. + +| Suite | C | AR goodput tok/s | Fixed DDTree tok/s | DDTree vs AR | DDTree AL | Acceptance | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | +| GSM8K | 3 | 18.88 | 24.46 | +29.4% | 5.92 | 37.0% | +| Math500 | 3 | 19.03 | 28.71 | +50.9% | 7.00 | 43.8% | +| GSM8K | 4 | 31.73 | 27.84 | -12.3% | 5.57 | 34.8% | +| Math500 | 4 | 31.81 | 32.92 | +3.1% | 7.10 | 44.4% | + +At C=4, five additional adaptive-DDTree repeats measured the same eight cases. + +| Suite | AR goodput tok/s | Adaptive DDTree tok/s | Adaptive vs AR | Speculative AL | Speculative acceptance | +| :--- | ---: | ---: | ---: | ---: | ---: | +| GSM8K | 31.73 | 30.36 | -4.4% | 4.83 | 30.2% | +| Math500 | 31.81 | 31.53 | -1.0% | 7.56 | 47.2% | + +The adaptive acceptance fields describe only the DDTree probes; they do not +measure the share of later tokens emitted by AR after fallback. Adaptive mode +substantially reduces fixed DDTree's GSM8K loss, but it did not beat AR on +either C=4 suite. + +All measured requests completed with exact 96-token accounting and no errors. +Ordered output hashes were not stable across the five repeats for either AR or +DDTree. Because this also affects AR, it is not a DDTree-only signal, but it is +still a concurrent reproducibility warning; these numbers support performance +routing experiments, not a deterministic-output or correctness claim. The +canonical summary reports this check explicitly as `Stable output`. + +For the measured setup, fixed blog-DDTree is the clear choice at C=1–3. At C=4 +the optimum becomes workload-dependent: use fixed DDTree for a known +high-acceptance Math-like workload, and AR for GSM8K or unknown/mixed traffic. +Adaptive DDTree is a lower-risk single speculative configuration, but AR still +had the highest or statistically close aggregate goodput in this C=4 screen. +The available C=5/C=10 HumanEval screen also favors AR; reasoning-suite evidence +has not yet been collected above C=4. These are routing observations for this +hardware, model pair, prompt mix, and output length—not universal defaults. + Run a quick screening repeat: ```bash diff --git a/harness/benchmarks/concurrency/attach_ddtree_metrics.py b/harness/benchmarks/concurrency/attach_ddtree_metrics.py new file mode 100755 index 000000000..d3d9ddeba --- /dev/null +++ b/harness/benchmarks/concurrency/attach_ddtree_metrics.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Attach and validate concurrent DDTree server telemetry for a measured report.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +MARKER = re.compile(r"\[concurrency-metrics\]\s+(\{.*\})") +COUNTERS = ("ddtree_steps", "ddtree_accepted_tokens", "target_forwards") + + +def load_metrics(path: Path) -> dict[str, dict[str, Any]]: + found: dict[str, dict[str, Any]] = {} + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + match = MARKER.search(line) + if not match: + continue + value = json.loads(match.group(1)) + response_id = value.get("response_id") or value.get("request_id") + if not isinstance(response_id, str) or not response_id: + raise ValueError("concurrency metric is missing response_id") + if response_id in found: + raise ValueError(f"duplicate concurrency metric for {response_id}") + for key in COUNTERS: + if ( + isinstance(value.get(key), bool) + or not isinstance(value.get(key), int) + or value[key] < 0 + ): + raise ValueError(f"{response_id}: invalid {key}") + found[response_id] = value + return found + + +def attach(report: dict[str, Any], metrics: dict[str, dict[str, Any]]) -> None: + requests = [ + request + for level in report.get("levels", []) + for wave in level.get("wave_results", []) + for request in wave.get("requests_detail", []) + if request.get("error") is None + ] + totals = {key: 0 for key in COUNTERS} + for request in requests: + response_id = request.get("response_id") + if not isinstance(response_id, str) or response_id not in metrics: + raise ValueError(f"missing concurrency metric for response {response_id!r}") + value = metrics[response_id] + if value["ddtree_steps"] <= 0: + raise ValueError(f"{response_id}: ddtree_steps must be positive") + request["ddtree_metrics"] = {key: value[key] for key in COUNTERS} + for key in COUNTERS: + totals[key] += value[key] + steps = totals["ddtree_steps"] + if not requests or steps <= 0: + raise ValueError("DDTree proof requires at least one successful request and step") + emitted = totals["ddtree_accepted_tokens"] + steps + report["ddtree_proof"] = { + **totals, + "speculative_emitted_tokens": emitted, + "mean_accepted_length": emitted / steps, + "acceptance_rate": emitted / (16 * steps), + "acceptance_denominator_tokens_per_step": 16, + "requests_proven": len(requests), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("server_log", type=Path) + args = parser.parse_args() + try: + report = json.loads(args.report.read_text(encoding="utf-8")) + attach(report, load_metrics(args.server_log)) + args.report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + proof = report["ddtree_proof"] + print( + f"DDTree AL={proof['mean_accepted_length']:.2f} " + f"acceptance={100 * proof['acceptance_rate']:.1f}% " + f"steps={proof['ddtree_steps']}" + ) + return 0 + except Exception as exc: + print(f"[ddtree-proof] error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py b/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py new file mode 100755 index 000000000..33f5f7b01 --- /dev/null +++ b/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Run a complete repository benchmark suite in fixed-concurrency waves.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import re +import statistics +import sys +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location( + "concurrent_benchmark", HERE / "concurrent_benchmark.py" +) +assert SPEC is not None and SPEC.loader is not None +base = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(base) + + +CONCURRENCY_METRICS_MARKER = re.compile(r"\[concurrency-metrics\]\s+(\{.*\})\s*$") +SERVER_DONE_MARKER = re.compile(r"\[server\] chat DONE\s+(\S+)") + + +def retired_response_ids(text: str) -> set[str]: + retired: set[str] = set() + for line in text.splitlines(): + marker = CONCURRENCY_METRICS_MARKER.search(line) + if marker: + try: + value = json.loads(marker.group(1)) + except json.JSONDecodeError: + value = None + if isinstance(value, dict): + response_id = value.get("response_id") or value.get("request_id") + if isinstance(response_id, str) and response_id: + retired.add(response_id) + done = SERVER_DONE_MARKER.search(line) + if done: + retired.add(done.group(1)) + return retired + + +def load_cases(path: Path) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + seen: set[str] = set() + for line_no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + record = json.loads(raw) + case_id = record.get("id") + if not isinstance(case_id, str) or not case_id or case_id in seen: + raise ValueError(f"{path}:{line_no}: missing or duplicate string id") + if isinstance(record.get("prompt"), str) and record["prompt"]: + prompt = record["prompt"] + else: + messages = record.get("messages") + if not isinstance(messages, list): + raise ValueError(f"{path}:{line_no}: 'messages' must be an array") + prompt = base.prompt_messages(messages) + seen.add(case_id) + cases.append({"id": case_id, "prompt": prompt}) + if not cases: + raise ValueError(f"{path}: no cases") + return cases + + +def wait_for_retirement(path: Path, response_ids: list[str], timeout: float) -> float: + started = time.perf_counter() + pending = set(response_ids) + deadline = started + timeout + while pending and time.perf_counter() < deadline: + if path.exists(): + text = path.read_text(encoding="utf-8", errors="replace") + pending -= retired_response_ids(text) + if pending: + time.sleep(0.05) + if pending: + raise TimeoutError(f"scheduler did not retire responses: {sorted(pending)}") + return time.perf_counter() - started + + +def aggregate_waves(clients: int, waves: list[dict[str, Any]]) -> dict[str, Any]: + details = [record for wave in waves for record in wave["requests_detail"]] + ok = [record for record in details if record["error"] is None] + completion = [record["completion_tokens"] for record in ok] + prompts = [record["prompt_tokens"] for record in ok] + rates = [record["request_decode_tok_s"] for record in ok] + ttfts = [record["ttft_s"] for record in ok] + completion_complete = bool(ok) and all(isinstance(value, int) for value in completion) + prompt_complete = bool(ok) and all(isinstance(value, int) for value in prompts) + wall = sum(wave["wall_s"] for wave in waves) + output_window_values = [wave.get("output_window_s") for wave in waves] + output_window = ( + sum(output_window_values) + if all(isinstance(value, (int, float)) for value in output_window_values) + else None + ) + prompt_window_values = [wave.get("prompt_to_first_token_s") for wave in waves] + prompt_window = ( + sum(prompt_window_values) + if all(isinstance(value, (int, float)) for value in prompt_window_values) + else None + ) + failures = sum(wave["failures"] for wave in waves) + return { + "clients": clients, + "waves": len(waves), + "requests": len(details), + "requests_ok": len(ok), + "failures": failures, + "wall_s": wall, + "completion_tokens_total": sum(completion) if completion_complete else None, + "token_count_complete": completion_complete, + "prompt_tokens_total": sum(prompts) if prompt_complete else None, + "prompt_tokens_min": min(prompts) if prompt_complete else None, + "prompt_tokens_max": max(prompts) if prompt_complete else None, + "prompt_token_count_complete": prompt_complete, + "prompt_to_first_token_s": prompt_window, + "prompt_tokens_per_s_to_first_token": ( + sum(prompts) / prompt_window + if prompt_complete and prompt_window is not None and prompt_window > 0 else None + ), + "aggregate_tok_s": sum(completion) / wall if completion_complete and wall > 0 else None, + "output_window_tok_s": ( + sum(completion) / output_window + if completion_complete and output_window is not None and output_window > 0 else None + ), + "request_decode_tok_s_median": ( + statistics.median(rates) + if len(rates) == len(ok) and ok else None + ), + "ttft_median_s": statistics.median(ttfts) if len(ttfts) == len(ok) and ok else None, + "ttft_max_s": max(ttfts) if len(ttfts) == len(ok) and ok else None, + "wave_results": waves, + } + + +def run(args: argparse.Namespace) -> int: + cases = load_cases(args.prompt_file) + if args.case_limit is not None: + if args.case_limit < 1 or args.case_limit > len(cases): + raise ValueError( + f"--case-limit must be between 1 and the suite size {len(cases)}" + ) + cases = cases[:args.case_limit] + if args.clients < 1 or len(cases) % args.clients: + raise ValueError( + f"suite size {len(cases)} must be divisible by --clients={args.clients}; " + "refusing a lower-concurrency tail wave" + ) + waves = [] + for offset in range(0, len(cases), args.clients): + selected = cases[offset:offset + args.clients] + wave = base.run_level(args.clients, args, [case["prompt"] for case in selected], 0) + for case, detail in zip(selected, wave["requests_detail"], strict=True): + detail["case_id"] = case["id"] + if args.retire_log and wave["failures"] == 0: + response_ids = [ + detail["response_id"] for detail in wave["requests_detail"] + if isinstance(detail.get("response_id"), str) + ] + if len(response_ids) != args.clients: + raise ValueError("successful wave is missing response IDs") + wait_s = wait_for_retirement(args.retire_log, response_ids, args.timeout) + wave["retirement_wait_s"] = wait_s + wave["wall_s"] += wait_s + waves.append(wave) + level = aggregate_waves(args.clients, waves) + level["fixed_token_workload_valid"] = ( + level["failures"] == 0 + and level["requests_ok"] == len(cases) + and all(wave["fixed_token_workload_valid"] is True for wave in waves) + ) if args.ignore_eos else None + metadata = ( + json.loads(args.server_metadata_json.read_text(encoding="utf-8")) + if args.server_metadata_json else {} + ) + report = { + "schema_version": 1, + "label": args.label, + "suite": args.suite, + "base_url": args.base_url, + "model": args.model, + "max_tokens": args.max_tokens, + "temperature": args.temperature, + "seed": args.seed, + "ignore_eos": args.ignore_eos, + "case_limit": args.case_limit, + "prompt_file_sha256": hashlib.sha256(args.prompt_file.read_bytes()).hexdigest(), + "server_metadata": metadata, + "levels": [level], + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(base.markdown(report), end="") + return 1 if base.level_failed(level, args.ignore_eos) else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:18080/v1") + parser.add_argument("--api-key", default="") + parser.add_argument("--model", default="luce-dflash") + parser.add_argument("--clients", type=int, required=True) + parser.add_argument("--suite", required=True) + parser.add_argument("--prompt-file", type=Path, required=True) + parser.add_argument("--case-limit", type=int) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--ignore-eos", action="store_true") + parser.add_argument("--timeout", type=float, default=1200.0) + parser.add_argument("--server-metadata-json", type=Path) + parser.add_argument("--retire-log", type=Path) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--label", default="") + return parser + + +def main() -> int: + try: + return run(build_parser().parse_args()) + except Exception as exc: + print(f"[canonical-bench] error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/concurrent_benchmark.py b/harness/benchmarks/concurrency/concurrent_benchmark.py index 7a1e638e4..f4ac51d52 100755 --- a/harness/benchmarks/concurrency/concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/concurrent_benchmark.py @@ -15,6 +15,8 @@ from pathlib import Path from typing import Any +PromptInput = str | list[dict[str, str]] + def sha256_text(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -63,7 +65,26 @@ def iter_sse_data(lines: Iterable[bytes]) -> Iterable[str]: yield "\n".join(data) -def stream_request(args: argparse.Namespace, prompt: str) -> dict[str, Any]: +def prompt_messages(prompt: PromptInput) -> list[dict[str, str]]: + if isinstance(prompt, str): + return [{"role": "user", "content": prompt}] + if not isinstance(prompt, list) or not prompt or any( + not isinstance(message, dict) + or not isinstance(message.get("role"), str) + or not isinstance(message.get("content"), str) + for message in prompt + ): + raise ValueError("messages must contain role/content strings") + return prompt + + +def prompt_input_sha256(prompt: PromptInput) -> str: + if isinstance(prompt, str): + return sha256_text(prompt) + return sha256_text(json.dumps(prompt, ensure_ascii=False, separators=(",", ":"))) + + +def stream_request(args: argparse.Namespace, prompt: PromptInput) -> dict[str, Any]: started = time.perf_counter() first = None content: list[str] = [] @@ -72,10 +93,11 @@ def stream_request(args: argparse.Namespace, prompt: str) -> dict[str, Any]: prompt_tokens = None finish_reason = None done_received = False + response_id = None error = None payload = { "model": args.model, - "messages": [{"role": "user", "content": prompt}], + "messages": prompt_messages(prompt), "stream": True, "stream_options": {"include_usage": True}, "max_tokens": args.max_tokens, @@ -98,6 +120,8 @@ def stream_request(args: argparse.Namespace, prompt: str) -> dict[str, Any]: done_received = True break event = json.loads(data) + if isinstance(event.get("id"), str): + response_id = event["id"] usage = event.get("usage") or {} if isinstance(usage.get("completion_tokens"), int): completion_tokens = usage["completion_tokens"] @@ -137,6 +161,7 @@ def stream_request(args: argparse.Namespace, prompt: str) -> dict[str, Any]: "decode_duration_s": decode_duration, "completion_tokens": completion_tokens, "prompt_tokens": prompt_tokens, "finish_reason": finish_reason, "done_received": done_received, "error": error, + "response_id": response_id, "content_sha256": sha256_text(output), "reasoning_content_sha256": sha256_text(reasoning_output), "content_chars": len(output), "reasoning_content_chars": len(reasoning_output), @@ -159,7 +184,7 @@ def worker(index: int) -> None: barrier.wait() record = stream_request(args, selected[index]) record["prompt_index"] = offset + index - record["prompt_sha256"] = sha256_text(selected[index]) + record["prompt_sha256"] = prompt_input_sha256(selected[index]) records[index] = record threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(clients)] diff --git a/harness/benchmarks/concurrency/generate_blog_prompts.py b/harness/benchmarks/concurrency/generate_blog_prompts.py new file mode 100755 index 000000000..ce2f7f1de --- /dev/null +++ b/harness/benchmarks/concurrency/generate_blog_prompts.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Write the exact raw HumanEval-style prompts used by scripts/bench_he.py.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "server" / "scripts")) +from bench_he import PROMPTS # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + args.out.parent.mkdir(parents=True, exist_ok=True) + records = [ + {"id": f"he_raw_{index:02d}", "suite": "he-raw", "name": name, + "prompt": prompt, "max_tokens": 128} + for index, (name, prompt) in enumerate(PROMPTS, 1) + ] + args.out.write_text( + "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records), + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/raw_prompt_identity.jinja b/harness/benchmarks/concurrency/raw_prompt_identity.jinja new file mode 100644 index 000000000..9451237a6 --- /dev/null +++ b/harness/benchmarks/concurrency/raw_prompt_identity.jinja @@ -0,0 +1 @@ +{%- for message in messages -%}{{ message.content }}{%- endfor -%} diff --git a/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh b/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh new file mode 100755 index 000000000..a7502712d --- /dev/null +++ b/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# Canonical repository suites under fixed-concurrency waves, including blog parity. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO="${REPO:-$(cd -- "$SCRIPT_DIR/../../.." && pwd -P)}" +CLIENT="${CLIENT:-$SCRIPT_DIR/canonical_concurrent_benchmark.py}" +BLOG_GENERATOR="${BLOG_GENERATOR:-$SCRIPT_DIR/generate_blog_prompts.py}" +DDTREE_PROOF="${DDTREE_PROOF:-$SCRIPT_DIR/attach_ddtree_metrics.py}" +SUMMARIZER="${SUMMARIZER:-$SCRIPT_DIR/summarize_canonical_concurrency.py}" +MODEL="${MODEL:-}" +DRAFT_MODEL="${DRAFT_MODEL:-}" +SERVER_BIN="${SERVER_BIN:-$REPO/server/build-hip/dflash_server}" +OUT="${OUT:-$REPO/.harness-runs/qwen36-canonical-$(date -u +%Y%m%dT%H%M%SZ)}" +SUITES="${SUITES:-he-raw,he,gsm,math,agent}" +VARIANTS="${VARIANTS:-ar}" +REPEATS="${REPEATS:-1}" +MAX_TOKENS="${MAX_TOKENS:-128}" +WARMUP_TOKENS="${WARMUP_TOKENS:-8}" +CLIENTS="${CLIENTS:-}" +CASE_LIMIT="${CASE_LIMIT:-}" +SLOTS="${SLOTS:-16}" +PORT="${PORT:-18116}" +HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-600}" +COOLDOWN_SECONDS="${COOLDOWN_SECONDS:-3}" + +usage() { + cat <<'EOF' +Usage: MODEL=/path/Qwen3.6-27B-Q4_K_M.gguf \ + run_qwen36_canonical_concurrency.sh + +Runs the exact raw 10-prompt HumanEval-style blog corpus plus the repository's +HumanEval-chat, GSM8K, Math500, and agent prompt suites. Each level processes +the entire suite in full fixed-width waves: C=1/2/5/10 for ten-case suites and +C=1/2/3/6 for the six-case agent suite. No prompt is duplicated within a run. + +The default AR variant measures the concurrent server path. +Set VARIANTS=blog-ddtree with a readable DRAFT_MODEL to run the optional +DDTree variant; it requires a server build that emits per-response +[concurrency-metrics] telemetry. The server still uses paged attention +because this script measures the concurrent implementation, including C=1. +EOF +} + +if [[ "${1:-}" == "--help" ]]; then usage; exit 0; fi +if [[ $# -ne 0 ]]; then usage >&2; exit 2; fi +for cmd in python3 curl sha256sum ldd; do command -v "$cmd" >/dev/null || { echo "missing $cmd" >&2; exit 2; }; done +[[ -r "$MODEL" ]] || { echo "set MODEL to a readable target GGUF" >&2; exit 2; } +[[ -x "$SERVER_BIN" ]] || { echo "missing server: $SERVER_BIN" >&2; exit 2; } +[[ "$REPEATS" =~ ^[1-9][0-9]*$ ]] || { echo "REPEATS must be positive" >&2; exit 2; } +[[ "$MAX_TOKENS" =~ ^[1-9][0-9]*$ ]] || { echo "MAX_TOKENS must be positive" >&2; exit 2; } +if ! [[ "$SLOTS" =~ ^[1-9][0-9]*$ ]] || (( SLOTS < 16 )); then + echo "SLOTS must be an integer >= 16" >&2 + exit 2 +fi +[[ ! -e "$OUT" ]] || { echo "refusing to overwrite $OUT" >&2; exit 2; } + +IFS=, read -r -a suite_list <<< "$SUITES" +IFS=, read -r -a variant_list <<< "$VARIANTS" +for suite in "${suite_list[@]}"; do + [[ "$suite" =~ ^(he-raw|he|gsm|math|agent)$ ]] || { echo "unknown suite $suite" >&2; exit 2; } +done +for variant in "${variant_list[@]}"; do + [[ "$variant" =~ ^(ar|blog-ddtree|adaptive-ddtree)$ ]] || { echo "unknown variant $variant" >&2; exit 2; } +done +if [[ "$VARIANTS" == *ddtree* ]]; then + [[ -r "$DRAFT_MODEL" ]] || { echo "blog-ddtree requires readable DRAFT_MODEL" >&2; exit 2; } +fi + +mkdir -p "$OUT/prompts" +python3 "$BLOG_GENERATOR" --out "$OUT/prompts/he-raw.jsonl" +for suite in he gsm math agent; do + cp "$REPO/harness/benchmarks/prompts/bench_${suite}.jsonl" "$OUT/prompts/$suite.jsonl" +done + +server_pid="" +stop_server() { + if [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; then + kill "$server_pid" 2>/dev/null || true + for _ in $(seq 1 30); do kill -0 "$server_pid" 2>/dev/null || break; sleep 1; done + kill -9 "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi + server_pid="" +} +trap stop_server EXIT +trap 'exit 130' INT TERM + +wait_health() { + local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) + while (( SECONDS < deadline )); do + kill -0 "$server_pid" 2>/dev/null || return 1 + curl -fsS --max-time 2 "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +levels_for_suite() { + if [[ -n "$CLIENTS" ]]; then echo "${CLIENTS//,/ }"; return; fi + [[ "$1" == agent ]] && echo "1 2 3 6" || echo "1 2 5 10" +} + +run_case() { + local repeat="$1" suite="$2" clients="$3" variant="$4" + local max_ctx=4096 + [[ "$suite" == agent ]] && max_ctx=32768 + local capacity=$((SLOTS * max_ctx)) + local case_dir="$OUT/$suite/c$clients/r$repeat/$variant" + mkdir -p "$case_dir" + local -a command launch client_common + command=("$SERVER_BIN" "$MODEL" --target-device hip:0 --paged-attention + --max-concurrency "$SLOTS" --kv-pool-tokens "$capacity" --max-ctx "$max_ctx" + --cache-type-k q4_0 --cache-type-v q4_0 --fa-window 0 + --prefix-cache-slots 0 --prefill-cache-slots 0 + --admission-coalesce-ms 5 --host 127.0.0.1 --port "$PORT" --model-name qwen36) + if [[ "$suite" == he-raw ]]; then + command+=(--chat-template-file "$SCRIPT_DIR/raw_prompt_identity.jinja") + fi + if [[ "$variant" == *ddtree ]]; then + local adaptive=0 + [[ "$variant" == adaptive-ddtree ]] && adaptive=1 + command+=(--draft "$DRAFT_MODEL" --draft-device hip:0 --ddtree + --ddtree-budget 22 --fast-rollback --draft-residency persistent) + launch=(env DFLASH_IGNORE_EOS=1 DFLASH27B_DRAFT_SWA=2048 DFLASH_DDTREE_ADAPTIVE="$adaptive" + DFLASH_MIN_TOKENS="$WARMUP_TOKENS" stdbuf -oL -eL "${command[@]}") + else + launch=(env DFLASH_IGNORE_EOS=1 DFLASH_MIN_TOKENS="$WARMUP_TOKENS" + stdbuf -oL -eL "${command[@]}") + fi + printf '%q ' "${launch[@]}" > "$case_dir/server-command.txt" + printf '\n' >> "$case_dir/server-command.txt" + python3 -c 'import hashlib,json,pathlib,sys +import subprocess +out,variant,suite,c,repeat,binary,model,draft,prompts,cmd,n_gen,repo=sys.argv[1:] +digest=lambda p: hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() if p else None +libs={} +for line in subprocess.run(["ldd",binary],text=True,capture_output=True).stdout.splitlines(): + fields=line.replace("=>"," ").split() + paths=[x for x in fields if x.startswith("/") and pathlib.Path(x).is_file()] + for lib in paths: libs[str(pathlib.Path(lib).resolve())]=digest(lib) +git_head=subprocess.run(["git","-C",repo,"rev-parse","HEAD"],text=True,capture_output=True).stdout.strip() or None +obj={"variant":variant,"suite":suite,"clients":int(c),"repeat":int(repeat), +"server_binary":str(pathlib.Path(binary).resolve()),"server_binary_sha256":digest(binary), +"model_sha256":digest(model),"draft_model_sha256":digest(draft), +"lucebox_git_head":git_head, +"resolved_shared_library_sha256":libs, +"prompt_file_sha256":digest(prompts),"server_command":pathlib.Path(cmd).read_text().strip(), +"blog_decode_settings":({"draft_quant":"Q8_0","draft_swa":2048,"ddtree_budget":22, +"fast_rollback":True,"adaptive":variant == "adaptive-ddtree","n_gen":int(n_gen)} if variant.endswith("ddtree") else None)} +pathlib.Path(out).write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")' \ + "$case_dir/server-metadata.json" "$variant" "$suite" "$clients" "$repeat" \ + "$SERVER_BIN" "$MODEL" "$([[ "$variant" == *ddtree ]] && echo "$DRAFT_MODEL")" \ + "$OUT/prompts/$suite.jsonl" "$case_dir/server-command.txt" "$MAX_TOKENS" "$REPO" + + echo "[canonical] $suite C=$clients repeat=$repeat variant=$variant" + "${launch[@]}" > "$case_dir/server.log" 2>&1 & + server_pid=$! + if ! wait_health; then tail -n 80 "$case_dir/server.log" >&2 || true; stop_server; return 1; fi + client_common=(--base-url "http://127.0.0.1:$PORT/v1" --model qwen36 + --suite "$suite" --clients "$clients" --prompt-file "$OUT/prompts/$suite.jsonl" + --temperature 0 --seed 1 --ignore-eos --timeout 1800 --retire-log "$case_dir/server.log") + [[ -n "$CASE_LIMIT" ]] && client_common+=(--case-limit "$CASE_LIMIT") + local status=0 + python3 "$CLIENT" "${client_common[@]}" --max-tokens "$WARMUP_TOKENS" \ + --out "$case_dir/warmup.json" --label "$variant $suite C=$clients warmup" \ + > "$case_dir/warmup.txt" || status=1 + if (( status == 0 )); then + python3 "$CLIENT" "${client_common[@]}" --max-tokens "$MAX_TOKENS" \ + --server-metadata-json "$case_dir/server-metadata.json" --out "$case_dir/bench.json" \ + --label "$variant $suite C=$clients repeat=$repeat" | tee "$case_dir/bench.txt" || status=1 + fi + stop_server + if (( status == 0 )) && [[ "$variant" == *ddtree ]]; then + python3 "$DDTREE_PROOF" "$case_dir/bench.json" "$case_dir/server.log" \ + | tee "$case_dir/ddtree-proof.txt" || status=1 + fi + sleep "$COOLDOWN_SECONDS" + return "$status" +} + +failures=0 +for ((repeat=1; repeat<=REPEATS; repeat++)); do + for suite in "${suite_list[@]}"; do + for clients in $(levels_for_suite "$suite"); do + (( clients <= SLOTS )) || { echo "C=$clients exceeds SLOTS=$SLOTS" >&2; exit 2; } + for variant in "${variant_list[@]}"; do + run_case "$repeat" "$suite" "$clients" "$variant" || failures=$((failures + 1)) + done + done + done +done +(( failures == 0 )) || { echo "$failures case(s) failed" >&2; exit 1; } +python3 "$SUMMARIZER" "$OUT" --out "$OUT/summary.md" +echo "[canonical] complete: $OUT" diff --git a/harness/benchmarks/concurrency/summarize_canonical_concurrency.py b/harness/benchmarks/concurrency/summarize_canonical_concurrency.py new file mode 100755 index 000000000..e250562f0 --- /dev/null +++ b/harness/benchmarks/concurrency/summarize_canonical_concurrency.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Summarize canonical concurrent benchmark reports into one comparison table.""" + +from __future__ import annotations + +import argparse +import json +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def fmt(value: Any, digits: int = 2) -> str: + return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" + + +def output_signature(report: dict[str, Any]) -> tuple[tuple[str, str, str], ...] | None: + level = report["levels"][0] + rows = [] + for wave in level.get("wave_results", []): + for request in wave.get("requests_detail", []): + try: + rows.append(( + request["case_id"], request["content_sha256"], + request["reasoning_content_sha256"], + )) + except KeyError: + return None + if len(rows) != level["requests"]: + return None + return tuple(rows) + + +def summarize(root: Path) -> str: + GroupKey = tuple[str, int, str, int | None, int] + FamilyKey = tuple[str, int, int | None, int] + groups: dict[GroupKey, list[dict[str, Any]]] = defaultdict(list) + repeat_ids: dict[GroupKey, set[int]] = defaultdict(set) + variant_repeats: dict[FamilyKey, dict[str, set[int]]] = defaultdict(dict) + for path in root.glob("*/c*/r*/*/bench.json"): + report = json.loads(path.read_text(encoding="utf-8")) + level = report["levels"][0] + metadata = report["server_metadata"] + suite = report.get("suite") + variant = metadata.get("variant") + clients = level.get("clients") + requests = level.get("requests") + repeat = metadata.get("repeat") + case_limit = report.get("case_limit") + if not isinstance(suite, str) or not isinstance(variant, str): + raise ValueError(f"invalid suite or variant metadata: {path}") + if isinstance(clients, bool) or not isinstance(clients, int) or clients < 1: + raise ValueError(f"invalid client count: {path}") + if isinstance(requests, bool) or not isinstance(requests, int) or requests < 1: + raise ValueError(f"invalid request count: {path}") + if case_limit is not None and ( + isinstance(case_limit, bool) or not isinstance(case_limit, int) or case_limit < 1 + ): + raise ValueError(f"invalid case_limit: {path}") + if isinstance(repeat, bool) or not isinstance(repeat, int) or repeat < 1: + raise ValueError(f"missing or invalid repeat id: {path}") + if level["failures"] or level["fixed_token_workload_valid"] is not True: + raise ValueError(f"invalid measured report: {path}") + if variant.endswith("ddtree"): + proof = report.get("ddtree_proof") + if not isinstance(proof, dict): + raise ValueError(f"missing positive DDTree proof: {path}") + steps = proof.get("ddtree_steps") + if ( + isinstance(steps, bool) or not isinstance(steps, int) or steps <= 0 + or proof.get("requests_proven") != requests + ): + raise ValueError(f"missing positive DDTree proof: {path}") + key = (suite, clients, variant, case_limit, requests) + if repeat in repeat_ids[key]: + raise ValueError(f"duplicate repeat id {repeat} for {key}") + repeat_ids[key].add(repeat) + groups[key].append(report) + family = (suite, clients, case_limit, requests) + variant_repeats.setdefault(family, {}).setdefault(variant, set()).add(repeat) + for family, variants in variant_repeats.items(): + if len({frozenset(repeats) for repeats in variants.values()}) > 1: + raise ValueError(f"mismatched repeat sets for {family}") + if not groups: + raise ValueError(f"no canonical reports under {root}") + lines = [ + "# Canonical Qwen3.6 concurrency benchmark", "", + "| Suite | C | Cases | Variant | Repeats | Goodput tok/s | Output-window tok/s | " + "Prompt tok/s to first | Request decode tok/s | TTFT median s | TTFT max s | " + "DDTree AL | Acceptance | Stable output |", + "| :--- | ---: | ---: | :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :---: |", + ] + sort_key = lambda item: ( + item[0][0], item[0][1], item[0][2], item[0][3] is not None, + item[0][3] or 0, item[0][4], + ) + for (suite, clients, variant, _case_limit, _requests), reports in sorted( + groups.items(), key=sort_key + ): + levels = [report["levels"][0] for report in reports] + proofs = [report.get("ddtree_proof") for report in reports] + med = lambda key: statistics.median(level[key] for level in levels) + al = ( + statistics.median(proof["mean_accepted_length"] for proof in proofs) + if all(proof is not None for proof in proofs) else None + ) + acceptance = ( + statistics.median(proof["acceptance_rate"] for proof in proofs) + if all(proof is not None for proof in proofs) else None + ) + acceptance_text = f"{100 * acceptance:.1f}%" if acceptance is not None else "n/a" + signatures = [output_signature(report) for report in reports] + complete = len(reports) >= 2 and all(signature is not None for signature in signatures) + stable = "YES" if complete and len(set(signatures)) == 1 else "NO" if complete else "n/a" + lines.append( + f"| {suite} | {clients} | {levels[0]['requests']} | {variant} | {len(reports)} | " + f"{fmt(med('aggregate_tok_s'))} | {fmt(med('output_window_tok_s'))} | " + f"{fmt(med('prompt_tokens_per_s_to_first_token'))} | " + f"{fmt(med('request_decode_tok_s_median'))} | " + f"{fmt(med('ttft_median_s'), 3)} | {fmt(med('ttft_max_s'), 3)} | " + f"{fmt(al)} | {acceptance_text} | {stable} |" + ) + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + text = summarize(args.root) + args.out.write_text(text, encoding="utf-8") + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py b/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py new file mode 100644 index 000000000..602975031 --- /dev/null +++ b/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Tests for attach_ddtree_metrics.py.""" + +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).with_name("attach_ddtree_metrics.py") +SPEC = importlib.util.spec_from_file_location("attach_ddtree_metrics", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +proof = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(proof) + + +class DDTreeProofTests(unittest.TestCase): + def test_attaches_acceptance_from_matched_requests(self) -> None: + report = {"levels": [{"wave_results": [{"requests_detail": [ + {"response_id": "a", "error": None}, + {"response_id": "b", "error": None}, + ]}]}]} + metrics = { + "a": {"ddtree_steps": 2, "ddtree_accepted_tokens": 8, "target_forwards": 4}, + "b": {"ddtree_steps": 3, "ddtree_accepted_tokens": 12, "target_forwards": 6}, + } + proof.attach(report, metrics) + self.assertEqual(report["ddtree_proof"]["ddtree_steps"], 5) + self.assertEqual(report["ddtree_proof"]["mean_accepted_length"], 5.0) + self.assertEqual(report["ddtree_proof"]["acceptance_rate"], 5 / 16) + + def test_missing_or_zero_step_proof_fails_closed(self) -> None: + report = {"levels": [{"wave_results": [{"requests_detail": [ + {"response_id": "a", "error": None}, + ]}]}]} + with self.assertRaisesRegex(ValueError, "missing concurrency metric"): + proof.attach(report, {}) + metrics = {"a": { + "ddtree_steps": 0, "ddtree_accepted_tokens": 0, "target_forwards": 1, + }} + with self.assertRaisesRegex(ValueError, "ddtree_steps must be positive"): + proof.attach(report, metrics) + + + def test_boolean_counters_are_not_integers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "server.log" + path.write_text("[concurrency-metrics] " + json.dumps({ + "response_id": "a", "ddtree_steps": True, + "ddtree_accepted_tokens": 1, "target_forwards": 1, + }) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid ddtree_steps"): + proof.load_metrics(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py b/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py new file mode 100644 index 000000000..5c7006e61 --- /dev/null +++ b/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Tests for canonical_concurrent_benchmark.py and blog prompt parity.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location( + "canonical_concurrent_benchmark", HERE / "canonical_concurrent_benchmark.py" +) +assert SPEC is not None and SPEC.loader is not None +benchmark = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(benchmark) + + +class CanonicalBenchmarkTests(unittest.TestCase): + def test_loads_raw_and_multi_message_cases(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cases.jsonl" + path.write_text( + json.dumps({"id": "raw", "prompt": "code"}) + "\n" + + json.dumps({"id": "chat", "messages": [ + {"role": "system", "content": "s"}, + {"role": "user", "content": "u"}, + ]}) + "\n", + encoding="utf-8", + ) + cases = benchmark.load_cases(path) + self.assertEqual(cases[0]["prompt"], "code") + self.assertEqual(cases[1]["prompt"][0]["role"], "system") + + def test_rejects_non_array_messages(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "invalid.jsonl" + path.write_text(json.dumps({"id": "bad", "messages": {"role": "user"}}) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "messages.*array"): + benchmark.load_cases(path) + + def test_full_suite_waves_have_no_tail_or_reuse(self) -> None: + cases = [ + json.dumps({"id": f"p{i}", "prompt": f"prompt {i}"}) + for i in range(10) + ] + seen = [] + + def fake_level(clients, args, prompts, offset): + self.assertEqual(clients, 5) + self.assertEqual(offset, 0) + seen.extend(prompts) + details = [{ + "error": None, "completion_tokens": 8, "prompt_tokens": 4, + "request_decode_tok_s": 2.0, "ttft_s": 0.1, + } for _ in prompts] + return { + "requests_detail": details, "failures": 0, "wall_s": 4.0, + "output_window_s": 3.0, "fixed_token_workload_valid": True, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt_file = root / "cases.jsonl" + prompt_file.write_text("\n".join(cases) + "\n", encoding="utf-8") + args = argparse.Namespace( + clients=5, prompt_file=prompt_file, suite="he", ignore_eos=True, + server_metadata_json=None, label="test", base_url="x", model="m", + max_tokens=8, temperature=0.0, seed=1, out=root / "report.json", + retire_log=None, case_limit=None, + ) + with mock.patch.object(benchmark.base, "run_level", side_effect=fake_level): + self.assertEqual(benchmark.run(args), 0) + report = json.loads(args.out.read_text(encoding="utf-8")) + self.assertEqual(seen, [f"prompt {i}" for i in range(10)]) + self.assertEqual(report["levels"][0]["waves"], 2) + self.assertEqual(report["levels"][0]["requests"], 10) + + def test_rejects_partial_tail_wave(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt_file = root / "cases.jsonl" + prompt_file.write_text("".join( + json.dumps({"id": f"p{i}", "prompt": "x"}) + "\n" + for i in range(10) + ), encoding="utf-8") + args = argparse.Namespace(clients=4, prompt_file=prompt_file, case_limit=None) + with self.assertRaisesRegex(ValueError, "lower-concurrency tail"): + benchmark.run(args) + + def test_case_limit_supports_three_full_c3_waves(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt_file = root / "cases.jsonl" + prompt_file.write_text("".join( + json.dumps({"id": f"p{i}", "prompt": f"x{i}"}) + "\n" + for i in range(10) + ), encoding="utf-8") + args = argparse.Namespace( + clients=3, case_limit=9, prompt_file=prompt_file, suite="he-raw", + ignore_eos=True, server_metadata_json=None, label="c3", base_url="x", + model="m", max_tokens=8, temperature=0.0, seed=1, + out=root / "report.json", retire_log=None, + ) + seen = [] + + def fake_level(clients, _args, prompts, offset): + self.assertEqual((clients, offset, len(prompts)), (3, 0, 3)) + seen.extend(prompts) + return { + "requests_detail": [{ + "error": None, "completion_tokens": 8, "prompt_tokens": 4, + "request_decode_tok_s": 2.0, "ttft_s": 0.1, + } for _ in prompts], + "failures": 0, "wall_s": 2.0, "output_window_s": 1.5, + "fixed_token_workload_valid": True, + } + + with mock.patch.object(benchmark.base, "run_level", side_effect=fake_level): + self.assertEqual(benchmark.run(args), 0) + report = json.loads(args.out.read_text(encoding="utf-8")) + self.assertEqual(seen, [f"x{i}" for i in range(9)]) + self.assertEqual(report["case_limit"], 9) + self.assertEqual(report["levels"][0]["waves"], 3) + + def test_retirement_wait_matches_every_response(self) -> None: + with tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "server.log" + log.write_text( + '[concurrency-metrics] {"ddtree_steps": 1, "request_id": "a"}\n' + '[server] chat DONE b ok=true\n', + encoding="utf-8", + ) + elapsed = benchmark.wait_for_retirement(log, ["a", "b"], 0.1) + self.assertGreaterEqual(elapsed, 0) + with self.assertRaisesRegex(TimeoutError, "did not retire"): + benchmark.wait_for_retirement(log, ["missing"], 0.01) + + def test_blog_generator_matches_bench_he_source(self) -> None: + generator_spec = importlib.util.spec_from_file_location( + "generate_blog_prompts", HERE / "generate_blog_prompts.py" + ) + assert generator_spec is not None and generator_spec.loader is not None + generator = importlib.util.module_from_spec(generator_spec) + generator_spec.loader.exec_module(generator) + self.assertEqual(len(generator.PROMPTS), 10) + source_spec = importlib.util.spec_from_file_location( + "bench_he_source", HERE.parents[2] / "server" / "scripts" / "bench_he.py" + ) + assert source_spec is not None and source_spec.loader is not None + source = importlib.util.module_from_spec(source_spec) + source_spec.loader.exec_module(source) + self.assertEqual(generator.PROMPTS, source.PROMPTS) + self.assertEqual( + (HERE / "raw_prompt_identity.jinja").read_text(encoding="utf-8"), + "{%- for message in messages -%}{{ message.content }}{%- endfor -%}\n", + ) + + def test_summary_keeps_suites_separate_and_reports_acceptance(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_canonical", HERE / "summarize_canonical_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for suite, variant, acceptance in (("he-raw", "blog-ddtree", 0.35), ("gsm", "ar", None)): + path = root / suite / "c1" / "r1" / variant / "bench.json" + path.parent.mkdir(parents=True) + report = { + "suite": suite, "case_limit": None, + "server_metadata": {"variant": variant, "repeat": 1}, + "levels": [{ + "clients": 1, "failures": 0, "fixed_token_workload_valid": True, + "requests": 1, + "aggregate_tok_s": 10.0, "output_window_tok_s": 11.0, + "prompt_tokens_per_s_to_first_token": 13.0, + "request_decode_tok_s_median": 12.0, + "ttft_median_s": 0.1, "ttft_max_s": 0.2, + }], + } + if acceptance is not None: + report["ddtree_proof"] = { + "ddtree_steps": 1, "requests_proven": 1, + "mean_accepted_length": 5.6, "acceptance_rate": acceptance, + } + path.write_text(json.dumps(report), encoding="utf-8") + text = summary.summarize(root) + self.assertIn("| he-raw | 1 | 1 | blog-ddtree", text) + self.assertIn("5.60 | 35.0%", text) + self.assertIn("| gsm | 1 | 1 | ar", text) + + def test_summary_reports_output_stability_by_case_id(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_canonical_stability", HERE / "summarize_canonical_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for repeat, content_hash in ((1, "same"), (2, "changed")): + path = root / "gsm" / "c1" / f"r{repeat}" / "ar" / "bench.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({ + "suite": "gsm", "case_limit": None, + "server_metadata": {"variant": "ar", "repeat": repeat}, + "levels": [{ + "clients": 1, "requests": 1, "failures": 0, + "fixed_token_workload_valid": True, "aggregate_tok_s": 10.0, + "output_window_tok_s": 11.0, + "prompt_tokens_per_s_to_first_token": 13.0, + "request_decode_tok_s_median": 12.0, + "ttft_median_s": 0.1, "ttft_max_s": 0.2, + "wave_results": [{"requests_detail": [{ + "case_id": "gsm_01", "content_sha256": content_hash, + "reasoning_content_sha256": "reasoning", + }]}], + }], + }), encoding="utf-8") + text = summary.summarize(root) + self.assertIn("| n/a | n/a | NO |", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/benchmarks/concurrency/test_concurrent_benchmark.py b/harness/benchmarks/concurrency/test_concurrent_benchmark.py index 201f3464f..e0b337dc4 100644 --- a/harness/benchmarks/concurrency/test_concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/test_concurrent_benchmark.py @@ -33,6 +33,10 @@ def test_prompt_selection_never_wraps(self) -> None: with self.assertRaisesRegex(ValueError, "refusing to reuse"): benchmark.request_prompts(["a", "b"], 2, 1) + def test_prompt_messages_rejects_non_array(self) -> None: + with self.assertRaisesRegex(ValueError, "messages must contain"): + benchmark.prompt_messages({"role": "user"}) + def test_level_uses_exact_usage_and_first_token_window(self) -> None: prompt_counts = iter((10, 30)) @@ -90,6 +94,34 @@ def __iter__(self): self.assertIsNotNone(record["request_decode_tok_s"]) self.assertEqual(record["content_sha256"], benchmark.sha256_text("one chunk")) + def test_stream_request_preserves_canonical_message_roles(self) -> None: + captured = {} + + class Response: + def __enter__(self): return self + def __exit__(self, *_args): return None + def __iter__(self): + return iter([ + b'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"length"}],"usage":{"prompt_tokens":4,"completion_tokens":1}}\n', + b"\n", b"data: [DONE]\n", b"\n", + ]) + + def fake_open(request, timeout): + captured["payload"] = __import__("json").loads(request.data) + return Response() + + args = argparse.Namespace( + model="m", max_tokens=1, temperature=0.0, seed=1, ignore_eos=True, + api_key="", base_url="http://localhost/v1", timeout=2.0, + ) + messages = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "user"}, + ] + with mock.patch.object(benchmark.urllib.request, "urlopen", side_effect=fake_open): + benchmark.stream_request(args, messages) + self.assertEqual(captured["payload"]["messages"], messages) + def test_stream_request_rejects_clean_eof_without_done(self) -> None: class Response: def __enter__(self): return self From 8adcf941ec34f8e7c5de7fc4cf458914781d0be7 Mon Sep 17 00:00:00 2001 From: Graffioh Date: Fri, 14 Aug 2026 11:47:08 +0000 Subject: [PATCH 3/3] refactor(bench): consolidate concurrency harness tools --- harness/benchmarks/concurrency/README.md | 104 ++++++- .../concurrency/attach_ddtree_metrics.py | 97 ------- .../canonical_concurrent_benchmark.py | 129 +++++++++ .../concurrency/concurrent_benchmark.py | 91 +++++- .../concurrency/generate_blog_prompts.py | 34 --- .../concurrency/generate_prompts.py | 151 ++++++++++ .../concurrency/generate_ragged_prompts.py | 78 ------ .../run_qwen36_canonical_concurrency.sh | 79 +++++- .../concurrency/run_qwen36_concurrency.sh | 156 +++++++++-- .../summarize_canonical_concurrency.py | 139 ---------- .../concurrency/summarize_concurrency.py | 224 ++++++++++++++- .../concurrency/test_attach_ddtree_metrics.py | 59 ---- .../test_canonical_concurrent_benchmark.py | 260 +++++++++++++++++- .../test_canonical_runner_policy.py | 151 ++++++++++ .../concurrency/test_concurrency_tools.py | 196 ++++++++++++- .../concurrency/test_concurrent_benchmark.py | 73 ++++- .../test_synthetic_runner_gpu_proof.py | 165 +++++++++++ 17 files changed, 1711 insertions(+), 475 deletions(-) delete mode 100755 harness/benchmarks/concurrency/attach_ddtree_metrics.py delete mode 100755 harness/benchmarks/concurrency/generate_blog_prompts.py create mode 100755 harness/benchmarks/concurrency/generate_prompts.py delete mode 100755 harness/benchmarks/concurrency/generate_ragged_prompts.py delete mode 100755 harness/benchmarks/concurrency/summarize_canonical_concurrency.py delete mode 100644 harness/benchmarks/concurrency/test_attach_ddtree_metrics.py create mode 100644 harness/benchmarks/concurrency/test_canonical_runner_policy.py create mode 100644 harness/benchmarks/concurrency/test_synthetic_runner_gpu_proof.py diff --git a/harness/benchmarks/concurrency/README.md b/harness/benchmarks/concurrency/README.md index 9f2adc7b1..af6bfd365 100644 --- a/harness/benchmarks/concurrency/README.md +++ b/harness/benchmarks/concurrency/README.md @@ -1,9 +1,9 @@ # Qwen3.6 concurrency benchmark This protocol measures the serving behavior targeted by packed continuous -prefill and concurrent decode. It is intentionally small: one streaming client, -one fresh-process runner, one deterministic prompt generator, and one summary -script. +prefill and concurrent decode. It has two user-facing runners that share the +streaming client, deterministic prompt generation, and summary tooling: +the paired ragged workload runner and the canonical-suite runner. ## Canonical and blog workloads @@ -13,6 +13,8 @@ runner: ```bash MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +GPU_DEVICE=1 \ +EXPECTED_GPU_ARCH=gfx1151 \ REPEATS=3 \ harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh ``` @@ -129,6 +131,8 @@ Run a quick screening repeat: ```bash MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +GPU_DEVICE=1 \ +EXPECTED_GPU_ARCH=gfx1151 \ LUCE_SERVER_BIN=server/build-hip/dflash_server \ LLAMA_SERVER_BIN=/path/to/llama-server \ harness/benchmarks/concurrency/run_qwen36_concurrency.sh @@ -138,6 +142,8 @@ Run a decode-heavy comparison with the same harness: ```bash MODEL=/path/to/Qwen3.6-27B-Q4_K_M.gguf \ +GPU_DEVICE=1 \ +EXPECTED_GPU_ARCH=gfx1151 \ LUCE_SERVER_BIN=server/build-hip/dflash_server \ LLAMA_SERVER_BIN=/path/to/llama-server \ WORKLOADS=short MAX_TOKENS=256 VARIANTS=luce-k8,llama REPEATS=3 \ @@ -151,13 +157,73 @@ discarded warmup at the same concurrency. The variants are: - `luce-k8`: packed prefill with up to eight concurrent prefills. - `luce-k1`: the same binary/configuration with packing width limited to one. -- `llama`: llama.cpp continuous batching with fixed `-b 2048 -ub 512`. +- `llama`: llama.cpp continuous batching with fixed `-b 2048 -ub 512` and + `--reasoning off --reasoning-format none`, matching Luce's non-thinking mode + and preserving raw immediate streaming for the fixed-token protocol. -The 29 generated prompts are disjoint cohorts for C1/C4/C8/C16. C4 and above -contain four substantial length strata while holding the mean target length -constant. The default short, medium, and long profiles target approximately -400, 1,000, and 3,000 input tokens per request. The client refuses to wrap or -reuse a prompt; reports retain the exact server-observed token counts. +Set `PREFILL_FIRST_BURST_STEPS=N` to benchmark the opt-in Lucebox TTFT policy. +The runner records the resolved value in both the server command and metadata. +`0` preserves continuous decode; positive `N` permits at most `N` consecutive +prefill-only traversals before a mandatory decode traversal. Screen `N=1` and +`N=2` against `N=0` before choosing a latency/active-decode tradeoff. + +`IDLE_PREFILL_TOKENS` exposes Luce's existing prefill-only traversal budget. +It defaults to the runtime's neutral value of 4096 and accepts 1 through 16384. +The upper bound is the largest currently effective K8 pure-prefill batch: +eight packed lanes times the 2048-token per-sequence cap. Both runners inject +the resolved value into Luce AR/DDTree launches and record it in metadata; +synthetic llama metadata records `null`, and llama never receives the +`DFLASH_IDLE_PREFILL_TOKENS` environment variable. + +Values above 4096 pair with a positive `PREFILL_FIRST_BURST_STEPS`: the burst +creates prefill-only traversal opportunities while the larger idle budget +allows those traversals to carry more than 4096 real prompt tokens. Mixed +decode+prefill traversals retain their separate mixed-token budgets. + +The default 30 generated prompts are disjoint cohorts for C2/C4/C8/C16. C2 +uses the shortest and longest strata; C4 and above contain all four substantial +length strata. Every cohort has the same mean target length. The default short, +medium, and long profiles target approximately 400, 1,000, and 3,000 input +tokens per request. Cohort IDs, local indices, and cumulative offsets are +recorded in the prompt manifest. The client refuses to wrap or reuse a prompt; +reports retain the exact prompt indices, prompt hashes, and server-observed +token counts. + +`CLIENTS` accepts any distinct positive levels. The generator appends a +deterministic, disjoint, mean-matched cohort for each requested level, while +leaving earlier cohorts unchanged. `SLOTS` stays at 16 for smaller screens and +automatically grows to the largest level, so this extends the matrix without +source changes: + +```bash +CLIENTS=2,4,8,16,32 GPU_DEVICE=1 EXPECTED_GPU_ARCH=gfx1151 \ +harness/benchmarks/concurrency/run_qwen36_concurrency.sh +``` + +Both runners default to physical ROCr `GPU_DEVICE=0`, expose only that device +to their server processes, and then address it as `hip:0`. On the dual-GPU +benchmark host, every Strix Halo comparison must set both `GPU_DEVICE=1` and +`EXPECTED_GPU_ARCH=gfx1151`, as in the commands above. Ambient GPU/backend +tuning variables are refused. + +The paired synthetic runner uses independent preflight and running-process +proofs, and aborts before warmup when any required proof is missing: + +- `gpu-identity.txt` is the matching line from `rocminfo` executed under the + isolated `ROCR_VISIBLE_DEVICES` value. This catches physical-index reorder. +- For llama.cpp, `llama-list-devices-command.txt` records the isolated command + and `llama-list-devices.txt` stores its raw output. The exact binary whose + path, hash, and version are in metadata must expose a `ROCmN:` device. +- After health, `server-gpu-proof.txt` comes from the actual server log. A + Lucebox case must report `gfx1151`. A llama.cpp case must report a positive + full offload in its revision-4cb22cd form, `offloaded N/N layers to GPU`; + partial offload, `0/0`, or CPU fallback fails the case. The llama server uses + `-lv 4` because that revision omits these startup lines at its default log + level; this changes logging verbosity, not the inference configuration. + +Keep all four evidence files with published llama.cpp result artifacts. Lucebox +cases do not produce the two `llama-list-devices*` files. Each server command +and metadata JSON also records the resolved isolation and expected architecture. The headline metric is aggregate output goodput: exact server-reported completion tokens divided by level wall time. It includes queueing, prefill, @@ -177,6 +243,26 @@ the latest first-token arrival; it is a useful prefill-facing metric but still includes admission, queueing, and transport. Report TTFT median/max alongside all throughput metrics. +Lucebox also supplies `usage.timings.prefilled_tokens` and +`usage.timings.prefill_ms` in the terminal streaming usage event. The client +copies those exact values into each request as +`server_native_prefilled_tokens` and `server_native_prefill_ms`, plus their +direct per-request rate. It never substitutes `usage.prompt_tokens` or client +TTFT when either native field is absent. + +For one concurrent ragged level, the native prefill window is +`max(start_offset_ms + server_native_prefill_ms)`. The level rate divides +`server_native_prefilled_tokens_total` by that common-origin window, avoiding +double-counting overlapping request intervals. Canonical suites run sequential +waves, so their top-level rate divides total native-prefilled tokens by the sum +of those per-wave windows. The report records the formula in +`server_native_prefill_metric` and exposes completeness flags. The summarizer +requires native telemetry on every repeat or none; partial repeats fail closed. + +Servers that do not expose both Luce timing fields, including the paired +llama.cpp baseline, display `n/a` for native metrics. The existing end-to-end +`Prompt tok/s to first` remains available and unchanged for both servers. + The K8-vs-K1 comparison is the causal packing ablation. The K8-vs-llama comparison is the product comparison. Five paired repeats, the exact command and hashes recorded in each case, zero failures, and a fixed declared output diff --git a/harness/benchmarks/concurrency/attach_ddtree_metrics.py b/harness/benchmarks/concurrency/attach_ddtree_metrics.py deleted file mode 100755 index d3d9ddeba..000000000 --- a/harness/benchmarks/concurrency/attach_ddtree_metrics.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -"""Attach and validate concurrent DDTree server telemetry for a measured report.""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -from pathlib import Path -from typing import Any - -MARKER = re.compile(r"\[concurrency-metrics\]\s+(\{.*\})") -COUNTERS = ("ddtree_steps", "ddtree_accepted_tokens", "target_forwards") - - -def load_metrics(path: Path) -> dict[str, dict[str, Any]]: - found: dict[str, dict[str, Any]] = {} - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - match = MARKER.search(line) - if not match: - continue - value = json.loads(match.group(1)) - response_id = value.get("response_id") or value.get("request_id") - if not isinstance(response_id, str) or not response_id: - raise ValueError("concurrency metric is missing response_id") - if response_id in found: - raise ValueError(f"duplicate concurrency metric for {response_id}") - for key in COUNTERS: - if ( - isinstance(value.get(key), bool) - or not isinstance(value.get(key), int) - or value[key] < 0 - ): - raise ValueError(f"{response_id}: invalid {key}") - found[response_id] = value - return found - - -def attach(report: dict[str, Any], metrics: dict[str, dict[str, Any]]) -> None: - requests = [ - request - for level in report.get("levels", []) - for wave in level.get("wave_results", []) - for request in wave.get("requests_detail", []) - if request.get("error") is None - ] - totals = {key: 0 for key in COUNTERS} - for request in requests: - response_id = request.get("response_id") - if not isinstance(response_id, str) or response_id not in metrics: - raise ValueError(f"missing concurrency metric for response {response_id!r}") - value = metrics[response_id] - if value["ddtree_steps"] <= 0: - raise ValueError(f"{response_id}: ddtree_steps must be positive") - request["ddtree_metrics"] = {key: value[key] for key in COUNTERS} - for key in COUNTERS: - totals[key] += value[key] - steps = totals["ddtree_steps"] - if not requests or steps <= 0: - raise ValueError("DDTree proof requires at least one successful request and step") - emitted = totals["ddtree_accepted_tokens"] + steps - report["ddtree_proof"] = { - **totals, - "speculative_emitted_tokens": emitted, - "mean_accepted_length": emitted / steps, - "acceptance_rate": emitted / (16 * steps), - "acceptance_denominator_tokens_per_step": 16, - "requests_proven": len(requests), - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("report", type=Path) - parser.add_argument("server_log", type=Path) - args = parser.parse_args() - try: - report = json.loads(args.report.read_text(encoding="utf-8")) - attach(report, load_metrics(args.server_log)) - args.report.write_text( - json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - proof = report["ddtree_proof"] - print( - f"DDTree AL={proof['mean_accepted_length']:.2f} " - f"acceptance={100 * proof['acceptance_rate']:.1f}% " - f"steps={proof['ddtree_steps']}" - ) - return 0 - except Exception as exc: - print(f"[ddtree-proof] error: {exc}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py b/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py index 33f5f7b01..46445a237 100755 --- a/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/canonical_concurrent_benchmark.py @@ -7,6 +7,7 @@ import hashlib import importlib.util import json +import math import re import statistics import sys @@ -25,6 +26,8 @@ CONCURRENCY_METRICS_MARKER = re.compile(r"\[concurrency-metrics\]\s+(\{.*\})\s*$") SERVER_DONE_MARKER = re.compile(r"\[server\] chat DONE\s+(\S+)") +DDTREE_METRICS_MARKER = re.compile(r"\[concurrency-metrics\]\s+(\{.*\})") +DDTREE_COUNTERS = ("ddtree_steps", "ddtree_accepted_tokens", "target_forwards") def retired_response_ids(text: str) -> set[str]: @@ -85,6 +88,73 @@ def wait_for_retirement(path: Path, response_ids: list[str], timeout: float) -> return time.perf_counter() - started +def load_ddtree_metrics(path: Path) -> dict[str, dict[str, Any]]: + found: dict[str, dict[str, Any]] = {} + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + match = DDTREE_METRICS_MARKER.search(line) + if not match: + continue + value = json.loads(match.group(1)) + response_id = value.get("response_id") or value.get("request_id") + if not isinstance(response_id, str) or not response_id: + raise ValueError("concurrency metric is missing response_id") + if response_id in found: + raise ValueError(f"duplicate concurrency metric for {response_id}") + for key in DDTREE_COUNTERS: + if ( + isinstance(value.get(key), bool) + or not isinstance(value.get(key), int) + or value[key] < 0 + ): + raise ValueError(f"{response_id}: invalid {key}") + found[response_id] = value + return found + + +def attach_ddtree_proof( + report: dict[str, Any], metrics: dict[str, dict[str, Any]] +) -> None: + levels = report.get("levels", []) + details = [ + request + for level in levels + for wave in level.get("wave_results", []) + for request in wave.get("requests_detail", []) + ] + if ( + any(level.get("failures") for level in levels) + or any(request.get("error") is not None for request in details) + ): + raise ValueError("DDTree proof requires a complete level with no failed requests") + requests = [ + request + for request in details + ] + totals = {key: 0 for key in DDTREE_COUNTERS} + for request in requests: + response_id = request.get("response_id") + if not isinstance(response_id, str) or response_id not in metrics: + raise ValueError(f"missing concurrency metric for response {response_id!r}") + value = metrics[response_id] + if value["ddtree_steps"] <= 0: + raise ValueError(f"{response_id}: ddtree_steps must be positive") + request["ddtree_metrics"] = {key: value[key] for key in DDTREE_COUNTERS} + for key in DDTREE_COUNTERS: + totals[key] += value[key] + steps = totals["ddtree_steps"] + if not requests or steps <= 0: + raise ValueError("DDTree proof requires at least one successful request and step") + emitted = totals["ddtree_accepted_tokens"] + steps + report["ddtree_proof"] = { + **totals, + "speculative_emitted_tokens": emitted, + "mean_accepted_length": emitted / steps, + "acceptance_rate": emitted / (16 * steps), + "acceptance_denominator_tokens_per_step": 16, + "requests_proven": len(requests), + } + + def aggregate_waves(clients: int, waves: list[dict[str, Any]]) -> dict[str, Any]: details = [record for wave in waves for record in wave["requests_detail"]] ok = [record for record in details if record["error"] is None] @@ -107,6 +177,30 @@ def aggregate_waves(clients: int, waves: list[dict[str, Any]]) -> dict[str, Any] if all(isinstance(value, (int, float)) for value in prompt_window_values) else None ) + native_tokens = [ + wave.get("server_native_prefilled_tokens_total") for wave in waves + ] + native_windows_ms = [ + wave.get("server_native_prefill_window_ms") for wave in waves + ] + native_token_count_complete = bool(waves) and all( + wave.get("server_native_prefill_token_count_complete") is True + and isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for wave, value in zip(waves, native_tokens, strict=True) + ) + native_timing_complete = bool(waves) and all( + wave.get("server_native_prefill_timing_complete") is True + and isinstance(value, (int, float)) and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + for wave, value in zip(waves, native_windows_ms, strict=True) + ) + native_tokens_total = ( + sum(native_tokens) if native_token_count_complete else None + ) + native_window_ms = ( + sum(native_windows_ms) if native_timing_complete else None + ) failures = sum(wave["failures"] for wave in waves) return { "clients": clients, @@ -135,6 +229,20 @@ def aggregate_waves(clients: int, waves: list[dict[str, Any]]) -> dict[str, Any] statistics.median(rates) if len(rates) == len(ok) and ok else None ), + "server_native_prefilled_tokens_total": native_tokens_total, + "server_native_prefill_window_ms": native_window_ms, + "server_native_prefill_token_count_complete": native_token_count_complete, + "server_native_prefill_timing_complete": native_timing_complete, + "server_native_prefill_tokens_per_s": ( + native_tokens_total * 1000.0 / native_window_ms + if native_tokens_total is not None + and native_window_ms is not None + and native_window_ms > 0 else None + ), + "server_native_prefill_metric": ( + "sum_prefilled_tokens_per_sum_wave_common_origin_" + "prefill_window_second" + ), "ttft_median_s": statistics.median(ttfts) if len(ttfts) == len(ok) and ok else None, "ttft_max_s": max(ttfts) if len(ttfts) == len(ok) and ok else None, "wave_results": waves, @@ -175,6 +283,8 @@ def run(args: argparse.Namespace) -> int: level["fixed_token_workload_valid"] = ( level["failures"] == 0 and level["requests_ok"] == len(cases) + and level["token_count_complete"] is True + and level["prompt_token_count_complete"] is True and all(wave["fixed_token_workload_valid"] is True for wave in waves) ) if args.ignore_eos else None metadata = ( @@ -196,9 +306,23 @@ def run(args: argparse.Namespace) -> int: "server_metadata": metadata, "levels": [level], } + ddtree_proof_attached = False + if getattr(args, "ddtree_proof", False): + if args.retire_log is None: + raise ValueError("--ddtree-proof requires --retire-log") + if level["failures"] == 0: + attach_ddtree_proof(report, load_ddtree_metrics(args.retire_log)) + ddtree_proof_attached = True args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(base.markdown(report), end="") + if ddtree_proof_attached: + proof = report["ddtree_proof"] + print( + f"DDTree AL={proof['mean_accepted_length']:.2f} " + f"acceptance={100 * proof['acceptance_rate']:.1f}% " + f"steps={proof['ddtree_steps']}" + ) return 1 if base.level_failed(level, args.ignore_eos) else 0 @@ -218,6 +342,11 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--timeout", type=float, default=1200.0) parser.add_argument("--server-metadata-json", type=Path) parser.add_argument("--retire-log", type=Path) + parser.add_argument( + "--ddtree-proof", + action="store_true", + help="validate and attach DDTree telemetry from --retire-log", + ) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--label", default="") return parser diff --git a/harness/benchmarks/concurrency/concurrent_benchmark.py b/harness/benchmarks/concurrency/concurrent_benchmark.py index f4ac51d52..c05303869 100755 --- a/harness/benchmarks/concurrency/concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/concurrent_benchmark.py @@ -6,6 +6,7 @@ import argparse import hashlib import json +import math import statistics import sys import threading @@ -16,6 +17,7 @@ from typing import Any PromptInput = str | list[dict[str, str]] +DEFAULT_CLIENT_LEVELS = (2, 4, 8, 16) def sha256_text(text: str) -> str: @@ -84,6 +86,27 @@ def prompt_input_sha256(prompt: PromptInput) -> str: return sha256_text(json.dumps(prompt, ensure_ascii=False, separators=(",", ":"))) +def native_prefill_values(usage: Any) -> tuple[int | None, float | None]: + """Return exact Luce usage.timings fields, without protocol fallbacks.""" + if not isinstance(usage, dict): + return None, None + timings = usage.get("timings") + if not isinstance(timings, dict): + return None, None + tokens = timings.get("prefilled_tokens") + prefill_ms = timings.get("prefill_ms") + if isinstance(tokens, bool) or not isinstance(tokens, int) or tokens < 0: + tokens = None + if ( + isinstance(prefill_ms, bool) + or not isinstance(prefill_ms, (int, float)) + or not math.isfinite(prefill_ms) + or prefill_ms < 0 + ): + prefill_ms = None + return tokens, float(prefill_ms) if prefill_ms is not None else None + + def stream_request(args: argparse.Namespace, prompt: PromptInput) -> dict[str, Any]: started = time.perf_counter() first = None @@ -91,6 +114,8 @@ def stream_request(args: argparse.Namespace, prompt: PromptInput) -> dict[str, A reasoning: list[str] = [] completion_tokens = None prompt_tokens = None + server_native_prefilled_tokens = None + server_native_prefill_ms = None finish_reason = None done_received = False response_id = None @@ -127,6 +152,11 @@ def stream_request(args: argparse.Namespace, prompt: PromptInput) -> dict[str, A completion_tokens = usage["completion_tokens"] if isinstance(usage.get("prompt_tokens"), int): prompt_tokens = usage["prompt_tokens"] + native_tokens, native_ms = native_prefill_values(usage) + if native_tokens is not None: + server_native_prefilled_tokens = native_tokens + if native_ms is not None: + server_native_prefill_ms = native_ms for choice in event.get("choices") or []: if choice.get("finish_reason") is not None: finish_reason = choice["finish_reason"] @@ -154,12 +184,21 @@ def stream_request(args: argparse.Namespace, prompt: PromptInput) -> dict[str, A if isinstance(completion_tokens, int) and completion_tokens > 0 and decode_duration is not None else None ) + server_native_prefill_tokens_per_s = ( + server_native_prefilled_tokens * 1000.0 / server_native_prefill_ms + if server_native_prefilled_tokens is not None + and server_native_prefill_ms is not None + and server_native_prefill_ms > 0 else None + ) return { "t_start": started, "t_first": first, "t_end": ended, "duration_s": ended - started, "ttft_s": first - started if first is not None else None, "decode_duration_s": decode_duration, "completion_tokens": completion_tokens, "prompt_tokens": prompt_tokens, + "server_native_prefilled_tokens": server_native_prefilled_tokens, + "server_native_prefill_ms": server_native_prefill_ms, + "server_native_prefill_tokens_per_s": server_native_prefill_tokens_per_s, "finish_reason": finish_reason, "done_received": done_received, "error": error, "response_id": response_id, "content_sha256": sha256_text(output), @@ -221,9 +260,33 @@ def worker(index: int) -> None: r["request_decode_tok_s"] for r in ok if r.get("request_decode_tok_s") is not None ] + native_prefilled_counts = [ + r.get("server_native_prefilled_tokens") for r in ok + ] + native_prefill_ms_values = [r.get("server_native_prefill_ms") for r in ok] + native_token_count_complete = bool(ok) and all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for value in native_prefilled_counts + ) + native_timing_complete = bool(ok) and all( + isinstance(value, (int, float)) and not isinstance(value, bool) + and math.isfinite(value) and value >= 0 + for value in native_prefill_ms_values + ) + native_prefilled_tokens_total = ( + sum(native_prefilled_counts) if native_token_count_complete else None + ) + native_prefill_ms_max = ( + max(native_prefill_ms_values) if native_timing_complete else None + ) + native_prefill_window_ms = ( + max(r["start_offset_s"] * 1000.0 + r["server_native_prefill_ms"] for r in ok) + if native_timing_complete else None + ) fixed_valid = ( failures == 0 and len(ok) == clients and completion_complete + and prompt_complete and all(v == args.max_tokens for v in completion_counts) ) if args.ignore_eos else None prompt_hashes = [r["prompt_sha256"] for r in ok] @@ -263,6 +326,20 @@ def worker(index: int) -> None: sum(prompt_counts) / first_window if prompt_complete and first_window is not None and first_window > 0 else None ), + "server_native_prefilled_tokens_total": native_prefilled_tokens_total, + "server_native_prefill_ms_max": native_prefill_ms_max, + "server_native_prefill_window_ms": native_prefill_window_ms, + "server_native_prefill_token_count_complete": native_token_count_complete, + "server_native_prefill_timing_complete": native_timing_complete, + "server_native_prefill_tokens_per_s": ( + native_prefilled_tokens_total * 1000.0 / native_prefill_window_ms + if native_prefilled_tokens_total is not None + and native_prefill_window_ms is not None + and native_prefill_window_ms > 0 else None + ), + "server_native_prefill_metric": ( + "sum_prefilled_tokens_per_common_origin_prefill_window_second" + ), "ttft_median_s": statistics.median(ttfts) if ttfts else None, "ttft_max_s": max(ttfts) if ttfts else None, "selected_prompt_set_sha256": digest(prompt_hashes), @@ -279,9 +356,10 @@ def markdown(report: dict[str, Any]) -> str: lines = [ f"# Concurrent benchmark — {report['label']}", "", "| C | Ok | Output goodput tok/s | Output-window tok/s | " - "Request decode tok/s | Prompt tok/s to first | Prompt range | " - "TTFT median s | TTFT max s | Wall s |", - "| ---: | ---: | ---: | ---: | ---: | ---: | :--- | ---: | ---: | ---: |", + "Request decode tok/s | Prompt tok/s to first | Native prefill tok/s | " + "Native prefill window ms | Prompt range | TTFT median s | TTFT max s | Wall s |", + "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :--- | " + "---: | ---: | ---: |", ] for level in report["levels"]: lines.append( @@ -290,6 +368,8 @@ def markdown(report: dict[str, Any]) -> str: f"{fmt(level['output_window_tok_s'])} | " f"{fmt(level['request_decode_tok_s_median'])} | " f"{fmt(level['prompt_tokens_per_s_to_first_token'])} | " + f"{fmt(level.get('server_native_prefill_tokens_per_s'))} | " + f"{fmt(level.get('server_native_prefill_window_ms'), '.1f')} | " f"{fmt(level['prompt_tokens_min'], '.0f')}–{fmt(level['prompt_tokens_max'], '.0f')} | " f"{fmt(level['ttft_median_s'], '.3f')} | {fmt(level['ttft_max_s'], '.3f')} | " f"{fmt(level['wall_s'])} |" @@ -329,9 +409,11 @@ def build_parser() -> argparse.ArgumentParser: def run(args: argparse.Namespace) -> int: - levels = args.client_levels or [1, 4, 8, 16] + levels = args.client_levels or list(DEFAULT_CLIENT_LEVELS) if any(level < 1 for level in levels): raise ValueError("--clients must be positive") + if len(set(levels)) != len(levels): + raise ValueError("--clients levels must be distinct") if args.prompt_offset < 0 or args.max_tokens < 1 or args.timeout <= 0: raise ValueError("invalid offset, max-tokens, or timeout") prompts = load_prompts(args.prompt_file) @@ -352,6 +434,7 @@ def run(args: argparse.Namespace) -> int: "model": args.model, "max_tokens": args.max_tokens, "temperature": args.temperature, "seed": args.seed, "ignore_eos": args.ignore_eos, "prompt_offset": args.prompt_offset, + "client_levels": levels, "prompt_file_sha256": hashlib.sha256(args.prompt_file.read_bytes()).hexdigest(), "server_metadata": metadata, "levels": results, } diff --git a/harness/benchmarks/concurrency/generate_blog_prompts.py b/harness/benchmarks/concurrency/generate_blog_prompts.py deleted file mode 100755 index ce2f7f1de..000000000 --- a/harness/benchmarks/concurrency/generate_blog_prompts.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -"""Write the exact raw HumanEval-style prompts used by scripts/bench_he.py.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -REPO = Path(__file__).resolve().parents[3] -sys.path.insert(0, str(REPO / "server" / "scripts")) -from bench_he import PROMPTS # noqa: E402 - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - args.out.parent.mkdir(parents=True, exist_ok=True) - records = [ - {"id": f"he_raw_{index:02d}", "suite": "he-raw", "name": name, - "prompt": prompt, "max_tokens": 128} - for index, (name, prompt) in enumerate(PROMPTS, 1) - ] - args.out.write_text( - "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records), - encoding="utf-8", - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/generate_prompts.py b/harness/benchmarks/concurrency/generate_prompts.py new file mode 100755 index 000000000..abafadab6 --- /dev/null +++ b/harness/benchmarks/concurrency/generate_prompts.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Generate deterministic prompt manifests for concurrency runs.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "server" / "scripts")) +from bench_he import PROMPTS # noqa: E402 + +RAGGED_PROFILES = { + "short": (250, 350, 450, 550), + "medium": (650, 850, 1150, 1350), + "long": (2000, 2600, 3400, 4000), +} + +DEFAULT_CLIENT_LEVELS = (2, 4, 8, 16) + +WORD_BANK = ( + "systems engineers compare latency throughput scheduling memory kernels queues " + "batches requests tokens caches pages attention arithmetic bandwidth occupancy " + "profiling measurement fairness reproducibility workloads concurrency admission " + "prefill decoding evidence tradeoffs implementation validation production service" +).split() + + +def parse_client_levels(value: str) -> tuple[int, ...]: + try: + levels = tuple(int(item) for item in value.split(",")) + except ValueError as exc: + raise ValueError("client levels must be comma-separated integers") from exc + if not levels or any(level < 1 for level in levels): + raise ValueError("client levels must be positive") + if len(set(levels)) != len(levels): + raise ValueError("client levels must be distinct") + return levels + + +def cohort_targets(strata: tuple[int, ...], clients: int) -> list[int]: + if clients < 1: + raise ValueError("clients must be positive") + if len(strata) != 4 or sum(strata) % len(strata): + raise ValueError("ragged profiles require four strata with an integer mean") + mean = sum(strata) // len(strata) + cycles, remainder = divmod(clients, len(strata)) + targets = list(strata) * cycles + if remainder == 1: + targets.append(mean) + elif remainder == 2: + targets.extend((strata[0], strata[-1])) + elif remainder == 3: + targets.extend((strata[0], mean, strata[-1])) + if len(targets) != clients or sum(targets) != clients * mean: + raise ValueError("profile strata must be symmetric around their mean") + return targets + + +def prompt_text(profile: str, cohort: str, index: int, target_words: int) -> str: + prefix = ( + f"Ragged benchmark {profile} cohort {cohort} request {index}. " + "Write a structured engineering analysis of the following observations, " + "including assumptions, likely bottlenecks, and a concise conclusion." + ).split() + words = list(prefix) + cursor = (index * 7 + target_words) % len(WORD_BANK) + while len(words) < target_words: + words.append(WORD_BANK[cursor % len(WORD_BANK)]) + cursor += 1 + return " ".join(words[:target_words]) + + +def build_ragged_records( + profile: str, client_levels: tuple[int, ...] = DEFAULT_CLIENT_LEVELS, +) -> list[dict[str, object]]: + strata = RAGGED_PROFILES[profile] + if not client_levels or any(level < 1 for level in client_levels): + raise ValueError("client levels must be positive") + if len(set(client_levels)) != len(client_levels): + raise ValueError("client levels must be distinct") + records: list[dict[str, object]] = [] + for clients in client_levels: + cohort = f"c{clients}" + cohort_offset = len(records) + for cohort_index, target in enumerate(cohort_targets(strata, clients)): + records.append({ + "id": f"{profile}-{cohort}-{cohort_index:04d}", + "cohort": cohort, + "cohort_clients": clients, + "cohort_index": cohort_index, + "cohort_offset": cohort_offset, + "stratum": strata.index(target) if target in strata else "mean", + "target_words": target, + "prompt": prompt_text(profile, cohort, cohort_index, target), + }) + return records + + +def build_raw_human_eval_records() -> list[dict[str, object]]: + return [ + { + "id": f"he_raw_{index:02d}", + "suite": "he-raw", + "name": name, + "prompt": prompt, + "max_tokens": 128, + } + for index, (name, prompt) in enumerate(PROMPTS, 1) + ] + + +def build_records( + profile: str, client_levels: tuple[int, ...] = DEFAULT_CLIENT_LEVELS, +) -> list[dict[str, object]]: + if profile == "he-raw": + return build_raw_human_eval_records() + return build_ragged_records(profile, client_levels) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profile", choices=["he-raw", *sorted(RAGGED_PROFILES)], required=True + ) + parser.add_argument( + "--clients", default=",".join(map(str, DEFAULT_CLIENT_LEVELS)), + help="comma-separated, distinct concurrency levels for disjoint cohorts", + ) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.out.exists(): + parser.error(f"refusing to overwrite {args.out}") + args.out.parent.mkdir(parents=True, exist_ok=True) + try: + client_levels = parse_client_levels(args.clients) + except ValueError as exc: + parser.error(str(exc)) + records = build_records(args.profile, client_levels) + args.out.write_text( + "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in records), + encoding="utf-8", + ) + print(f"wrote {len(records)} prompts to {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/generate_ragged_prompts.py b/harness/benchmarks/concurrency/generate_ragged_prompts.py deleted file mode 100755 index e3dce8c49..000000000 --- a/harness/benchmarks/concurrency/generate_ragged_prompts.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a small deterministic ragged-prompt manifest for concurrency runs.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -PROFILES = { - "short": (250, 350, 450, 550), - "medium": (650, 850, 1150, 1350), - "long": (2000, 2600, 3400, 4000), -} - -WORD_BANK = ( - "systems engineers compare latency throughput scheduling memory kernels queues " - "batches requests tokens caches pages attention arithmetic bandwidth occupancy " - "profiling measurement fairness reproducibility workloads concurrency admission " - "prefill decoding evidence tradeoffs implementation validation production service" -).split() - - -def prompt_text(profile: str, cohort: str, index: int, target_words: int) -> str: - prefix = ( - f"Ragged benchmark {profile} cohort {cohort} request {index}. " - "Write a structured engineering analysis of the following observations, " - "including assumptions, likely bottlenecks, and a concise conclusion." - ).split() - words = list(prefix) - cursor = (index * 7 + target_words) % len(WORD_BANK) - while len(words) < target_words: - words.append(WORD_BANK[cursor % len(WORD_BANK)]) - cursor += 1 - return " ".join(words[:target_words]) - - -def build_records(profile: str) -> list[dict[str, object]]: - strata = PROFILES[profile] - layout = [ - ("c1", [sum(strata) // len(strata)]), - ("c4", list(strata)), - ("c8", list(strata) * 2), - ("c16", list(strata) * 4), - ] - records: list[dict[str, object]] = [] - for cohort, targets in layout: - for target in targets: - index = len(records) - records.append({ - "id": f"{profile}-{index:02d}", - "cohort": cohort, - "stratum": strata.index(target) if target in strata else "mean", - "target_words": target, - "prompt": prompt_text(profile, cohort, index, target), - }) - return records - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--profile", choices=sorted(PROFILES), required=True) - parser.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - if args.out.exists(): - parser.error(f"refusing to overwrite {args.out}") - args.out.parent.mkdir(parents=True, exist_ok=True) - records = build_records(args.profile) - args.out.write_text( - "".join(json.dumps(row, sort_keys=True) + "\n" for row in records), - encoding="utf-8", - ) - print(f"wrote {len(records)} prompts to {args.out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh b/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh index a7502712d..b79a55d76 100755 --- a/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh +++ b/harness/benchmarks/concurrency/run_qwen36_canonical_concurrency.sh @@ -5,9 +5,8 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" REPO="${REPO:-$(cd -- "$SCRIPT_DIR/../../.." && pwd -P)}" CLIENT="${CLIENT:-$SCRIPT_DIR/canonical_concurrent_benchmark.py}" -BLOG_GENERATOR="${BLOG_GENERATOR:-$SCRIPT_DIR/generate_blog_prompts.py}" -DDTREE_PROOF="${DDTREE_PROOF:-$SCRIPT_DIR/attach_ddtree_metrics.py}" -SUMMARIZER="${SUMMARIZER:-$SCRIPT_DIR/summarize_canonical_concurrency.py}" +GENERATOR="${GENERATOR:-$SCRIPT_DIR/generate_prompts.py}" +SUMMARIZER="${SUMMARIZER:-$SCRIPT_DIR/summarize_concurrency.py}" MODEL="${MODEL:-}" DRAFT_MODEL="${DRAFT_MODEL:-}" SERVER_BIN="${SERVER_BIN:-$REPO/server/build-hip/dflash_server}" @@ -20,9 +19,13 @@ WARMUP_TOKENS="${WARMUP_TOKENS:-8}" CLIENTS="${CLIENTS:-}" CASE_LIMIT="${CASE_LIMIT:-}" SLOTS="${SLOTS:-16}" +GPU_DEVICE="${GPU_DEVICE:-0}" +EXPECTED_GPU_ARCH="${EXPECTED_GPU_ARCH:-}" PORT="${PORT:-18116}" HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-600}" COOLDOWN_SECONDS="${COOLDOWN_SECONDS:-3}" +PREFILL_FIRST_BURST_STEPS="${PREFILL_FIRST_BURST_STEPS:-0}" +IDLE_PREFILL_TOKENS="${IDLE_PREFILL_TOKENS:-4096}" usage() { cat <<'EOF' @@ -39,6 +42,19 @@ Set VARIANTS=blog-ddtree with a readable DRAFT_MODEL to run the optional DDTree variant; it requires a server build that emits per-response [concurrency-metrics] telemetry. The server still uses paged attention because this script measures the concurrent implementation, including C=1. +GPU_DEVICE is the physical ROCr device exposed exclusively to the server and +defaults to device 0. Pass GPU_DEVICE=1 for Strix Halo on this dual-GPU +benchmark host. The resolved value is stored in each case's command and metadata. +Set EXPECTED_GPU_ARCH=gfx1151 to require that literal architecture in the +server startup log; an empty value disables the check. The expectation is +stored in each case's metadata alongside the selected physical device. +PREFILL_FIRST_BURST_STEPS forwards the experimental Lucebox TTFT policy; 0 +preserves continuous decode, while positive N permits at most N consecutive +prefill-only traversals before a mandatory decode traversal. Valid values are +0 through 1024, matching the server-side limit. +IDLE_PREFILL_TOKENS forwards Luce's idle/prefill-only token budget and defaults +to 4096. Valid values are 1..16384, the current maximum effective K8 pure +prefill batch (8 lanes x 2048 tokens). EOF } @@ -49,11 +65,27 @@ for cmd in python3 curl sha256sum ldd; do command -v "$cmd" >/dev/null || { echo [[ -x "$SERVER_BIN" ]] || { echo "missing server: $SERVER_BIN" >&2; exit 2; } [[ "$REPEATS" =~ ^[1-9][0-9]*$ ]] || { echo "REPEATS must be positive" >&2; exit 2; } [[ "$MAX_TOKENS" =~ ^[1-9][0-9]*$ ]] || { echo "MAX_TOKENS must be positive" >&2; exit 2; } +[[ -z "$EXPECTED_GPU_ARCH" || "$EXPECTED_GPU_ARCH" =~ ^gfx[0-9a-f]+$ ]] || { echo "EXPECTED_GPU_ARCH must be empty or a gfx architecture" >&2; exit 2; } +[[ "$GPU_DEVICE" =~ ^[0-9]+$ ]] || { echo "GPU_DEVICE must be a physical ROCr device index" >&2; exit 2; } +if ! [[ "$PREFILL_FIRST_BURST_STEPS" =~ ^[0-9]{1,4}$ ]] || (( 10#$PREFILL_FIRST_BURST_STEPS > 1024 )); then + echo "PREFILL_FIRST_BURST_STEPS must be an integer in range 0..1024" >&2 + exit 2 +fi +if ! [[ "$IDLE_PREFILL_TOKENS" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#$IDLE_PREFILL_TOKENS > 16384 )); then + echo "IDLE_PREFILL_TOKENS must be an integer in range 1..16384" >&2 + exit 2 +fi if ! [[ "$SLOTS" =~ ^[1-9][0-9]*$ ]] || (( SLOTS < 16 )); then echo "SLOTS must be an integer >= 16" >&2 exit 2 fi [[ ! -e "$OUT" ]] || { echo "refusing to overwrite $OUT" >&2; exit 2; } +ambient_tuning="$(env | grep -E '^(GGML_|DFLASH_|LUCE_|HIP_|ROCR_|HSA_|LD_PRELOAD=|LD_LIBRARY_PATH=)' || true)" +if [[ -n "$ambient_tuning" ]]; then + echo "refusing ambient GPU/backend tuning variables:" >&2 + echo "$ambient_tuning" >&2 + exit 2 +fi IFS=, read -r -a suite_list <<< "$SUITES" IFS=, read -r -a variant_list <<< "$VARIANTS" @@ -68,7 +100,7 @@ if [[ "$VARIANTS" == *ddtree* ]]; then fi mkdir -p "$OUT/prompts" -python3 "$BLOG_GENERATOR" --out "$OUT/prompts/he-raw.jsonl" +python3 "$GENERATOR" --profile he-raw --out "$OUT/prompts/he-raw.jsonl" for suite in he gsm math agent; do cp "$REPO/harness/benchmarks/prompts/bench_${suite}.jsonl" "$OUT/prompts/$suite.jsonl" done @@ -108,7 +140,7 @@ run_case() { local capacity=$((SLOTS * max_ctx)) local case_dir="$OUT/$suite/c$clients/r$repeat/$variant" mkdir -p "$case_dir" - local -a command launch client_common + local -a command launch client_common bench_options command=("$SERVER_BIN" "$MODEL" --target-device hip:0 --paged-attention --max-concurrency "$SLOTS" --kv-pool-tokens "$capacity" --max-ctx "$max_ctx" --cache-type-k q4_0 --cache-type-v q4_0 --fa-window 0 @@ -122,17 +154,21 @@ run_case() { [[ "$variant" == adaptive-ddtree ]] && adaptive=1 command+=(--draft "$DRAFT_MODEL" --draft-device hip:0 --ddtree --ddtree-budget 22 --fast-rollback --draft-residency persistent) - launch=(env DFLASH_IGNORE_EOS=1 DFLASH27B_DRAFT_SWA=2048 DFLASH_DDTREE_ADAPTIVE="$adaptive" + launch=(env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" DFLASH_IGNORE_EOS=1 DFLASH27B_DRAFT_SWA=2048 DFLASH_DDTREE_ADAPTIVE="$adaptive" + DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS" + DFLASH_IDLE_PREFILL_TOKENS="$IDLE_PREFILL_TOKENS" DFLASH_MIN_TOKENS="$WARMUP_TOKENS" stdbuf -oL -eL "${command[@]}") else - launch=(env DFLASH_IGNORE_EOS=1 DFLASH_MIN_TOKENS="$WARMUP_TOKENS" + launch=(env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" DFLASH_IGNORE_EOS=1 DFLASH_MIN_TOKENS="$WARMUP_TOKENS" + DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS" + DFLASH_IDLE_PREFILL_TOKENS="$IDLE_PREFILL_TOKENS" stdbuf -oL -eL "${command[@]}") fi printf '%q ' "${launch[@]}" > "$case_dir/server-command.txt" printf '\n' >> "$case_dir/server-command.txt" python3 -c 'import hashlib,json,pathlib,sys import subprocess -out,variant,suite,c,repeat,binary,model,draft,prompts,cmd,n_gen,repo=sys.argv[1:] +out,variant,suite,c,repeat,binary,model,draft,prompts,cmd,n_gen,repo,rocr_visible,slots,prefill_first_burst_steps,expected_gpu_arch,idle_prefill_tokens=sys.argv[1:] digest=lambda p: hashlib.sha256(pathlib.Path(p).read_bytes()).hexdigest() if p else None libs={} for line in subprocess.run(["ldd",binary],text=True,capture_output=True).stdout.splitlines(): @@ -146,35 +182,48 @@ obj={"variant":variant,"suite":suite,"clients":int(c),"repeat":int(repeat), "lucebox_git_head":git_head, "resolved_shared_library_sha256":libs, "prompt_file_sha256":digest(prompts),"server_command":pathlib.Path(cmd).read_text().strip(), +"rocr_visible_devices":rocr_visible,"server_slots":int(slots), +"prefill_first_burst_steps":int(prefill_first_burst_steps), +"idle_prefill_tokens":int(idle_prefill_tokens), +"expected_gpu_arch":expected_gpu_arch or None, "blog_decode_settings":({"draft_quant":"Q8_0","draft_swa":2048,"ddtree_budget":22, "fast_rollback":True,"adaptive":variant == "adaptive-ddtree","n_gen":int(n_gen)} if variant.endswith("ddtree") else None)} pathlib.Path(out).write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")' \ "$case_dir/server-metadata.json" "$variant" "$suite" "$clients" "$repeat" \ "$SERVER_BIN" "$MODEL" "$([[ "$variant" == *ddtree ]] && echo "$DRAFT_MODEL")" \ - "$OUT/prompts/$suite.jsonl" "$case_dir/server-command.txt" "$MAX_TOKENS" "$REPO" + "$OUT/prompts/$suite.jsonl" "$case_dir/server-command.txt" "$MAX_TOKENS" "$REPO" \ + "$GPU_DEVICE" "$SLOTS" "$PREFILL_FIRST_BURST_STEPS" "$EXPECTED_GPU_ARCH" "$IDLE_PREFILL_TOKENS" echo "[canonical] $suite C=$clients repeat=$repeat variant=$variant" "${launch[@]}" > "$case_dir/server.log" 2>&1 & server_pid=$! if ! wait_health; then tail -n 80 "$case_dir/server.log" >&2 || true; stop_server; return 1; fi + if [[ -n "$EXPECTED_GPU_ARCH" ]]; then + if ! grep -F -- "$EXPECTED_GPU_ARCH" "$case_dir/server.log" > "$case_dir/gpu-identity.txt"; then + echo "server log does not identify expected GPU architecture: $EXPECTED_GPU_ARCH" >&2 + tail -n 80 "$case_dir/server.log" >&2 || true + stop_server + sleep "$COOLDOWN_SECONDS" + return 1 + fi + fi client_common=(--base-url "http://127.0.0.1:$PORT/v1" --model qwen36 --suite "$suite" --clients "$clients" --prompt-file "$OUT/prompts/$suite.jsonl" --temperature 0 --seed 1 --ignore-eos --timeout 1800 --retire-log "$case_dir/server.log") [[ -n "$CASE_LIMIT" ]] && client_common+=(--case-limit "$CASE_LIMIT") + bench_options=() + [[ "$variant" == *ddtree ]] && bench_options+=(--ddtree-proof) local status=0 python3 "$CLIENT" "${client_common[@]}" --max-tokens "$WARMUP_TOKENS" \ --out "$case_dir/warmup.json" --label "$variant $suite C=$clients warmup" \ > "$case_dir/warmup.txt" || status=1 if (( status == 0 )); then python3 "$CLIENT" "${client_common[@]}" --max-tokens "$MAX_TOKENS" \ - --server-metadata-json "$case_dir/server-metadata.json" --out "$case_dir/bench.json" \ + --server-metadata-json "$case_dir/server-metadata.json" "${bench_options[@]}" \ + --out "$case_dir/bench.json" \ --label "$variant $suite C=$clients repeat=$repeat" | tee "$case_dir/bench.txt" || status=1 fi stop_server - if (( status == 0 )) && [[ "$variant" == *ddtree ]]; then - python3 "$DDTREE_PROOF" "$case_dir/bench.json" "$case_dir/server.log" \ - | tee "$case_dir/ddtree-proof.txt" || status=1 - fi sleep "$COOLDOWN_SECONDS" return "$status" } @@ -191,5 +240,5 @@ for ((repeat=1; repeat<=REPEATS; repeat++)); do done done (( failures == 0 )) || { echo "$failures case(s) failed" >&2; exit 1; } -python3 "$SUMMARIZER" "$OUT" --out "$OUT/summary.md" +python3 "$SUMMARIZER" "$OUT" --format canonical --out "$OUT/summary.md" echo "[canonical] complete: $OUT" diff --git a/harness/benchmarks/concurrency/run_qwen36_concurrency.sh b/harness/benchmarks/concurrency/run_qwen36_concurrency.sh index d809815bb..09a4e42c7 100755 --- a/harness/benchmarks/concurrency/run_qwen36_concurrency.sh +++ b/harness/benchmarks/concurrency/run_qwen36_concurrency.sh @@ -5,7 +5,7 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" REPO="${REPO:-$(cd -- "$SCRIPT_DIR/../../.." && pwd -P)}" CLIENT="${CLIENT:-$SCRIPT_DIR/concurrent_benchmark.py}" -GENERATOR="${GENERATOR:-$SCRIPT_DIR/generate_ragged_prompts.py}" +GENERATOR="${GENERATOR:-$SCRIPT_DIR/generate_prompts.py}" SUMMARIZER="${SUMMARIZER:-$SCRIPT_DIR/summarize_concurrency.py}" MODEL="${MODEL:-}" @@ -15,23 +15,42 @@ OUT="${OUT:-$REPO/.harness-runs/qwen36-concurrency-$(date -u +%Y%m%dT%H%M%SZ)}" REPEATS="${REPEATS:-1}" WORKLOADS="${WORKLOADS:-short,medium,long}" VARIANTS="${VARIANTS:-luce-k8,luce-k1,llama}" -CLIENTS="${CLIENTS:-1,4,8,16}" +CLIENTS="${CLIENTS:-2,4,8,16}" +SLOTS="${SLOTS:-}" +GPU_DEVICE="${GPU_DEVICE:-0}" +EXPECTED_GPU_ARCH="${EXPECTED_GPU_ARCH:-}" PORT="${PORT:-18114}" COOLDOWN_SECONDS="${COOLDOWN_SECONDS:-3}" HEALTH_TIMEOUT_SECONDS="${HEALTH_TIMEOUT_SECONDS:-600}" MAX_TOKENS="${MAX_TOKENS:-64}" WARMUP_TOKENS="${WARMUP_TOKENS:-8}" -SLOTS=16 +PREFILL_FIRST_BURST_STEPS="${PREFILL_FIRST_BURST_STEPS:-0}" +IDLE_PREFILL_TOKENS="${IDLE_PREFILL_TOKENS:-4096}" usage() { cat <<'EOF' Usage: MODEL=/path/model.gguf [REPEATS=5] run_qwen36_concurrency.sh Runs fresh-server, same-concurrency warmup + measurement cases for luce-k8, -luce-k1, and llama at C=1/4/8/16. Defaults to one repeat for screening; use at +luce-k1, and llama at C=2/4/8/16. Defaults to one repeat for screening; use at least five paired repeats for publication. For a decode-heavy comparison, set WORKLOADS=short MAX_TOKENS=256 VARIANTS=luce-k8,llama. OUT must not already -exist. +exist. GPU_DEVICE is the physical ROCr device exposed exclusively to both +servers and defaults to device 0. Pass GPU_DEVICE=1 for Strix Halo on this +dual-GPU benchmark host. CLIENTS can contain any distinct positive levels; +SLOTS defaults to at least 16 and grows to the largest requested level. Set +EXPECTED_GPU_ARCH (for example gfx1151) to fail closed unless isolated +rocminfo identifies that architecture. After health, Luce must report that +architecture in its own log. llama.cpp must expose a ROCm device through this +exact binary's --list-devices output and report a positive full GPU offload +(offloaded N/N layers). The runner uses -lv 4 so revision 4cb22cd emits those +startup proof lines. Evidence for every check is retained in the case dir. +PREFILL_FIRST_BURST_STEPS forwards the experimental Lucebox TTFT policy; 0 +preserves continuous decode, while positive N permits at most N consecutive +prefill-only traversals before a mandatory decode traversal. +IDLE_PREFILL_TOKENS forwards Luce's idle/prefill-only token budget and defaults +to 4096. Valid values are 1..16384, the current maximum effective K8 pure +prefill batch (8 lanes x 2048 tokens); llama.cpp does not receive this setting. EOF } @@ -42,7 +61,17 @@ for cmd in python3 curl sha256sum; do command -v "$cmd" >/dev/null || { echo "mi [[ -x "$LUCE_SERVER_BIN" ]] || { echo "missing Lucebox server: $LUCE_SERVER_BIN" >&2; exit 2; } [[ -x "$LLAMA_SERVER_BIN" ]] || { echo "missing llama.cpp server: $LLAMA_SERVER_BIN" >&2; exit 2; } [[ "$REPEATS" =~ ^[1-9][0-9]*$ ]] || { echo "REPEATS must be positive" >&2; exit 2; } +[[ -z "$EXPECTED_GPU_ARCH" || "$EXPECTED_GPU_ARCH" =~ ^gfx[0-9a-f]+$ ]] || { echo "EXPECTED_GPU_ARCH must be empty or a gfx architecture" >&2; exit 2; } +[[ "$GPU_DEVICE" =~ ^[0-9]+$ ]] || { echo "GPU_DEVICE must be a physical ROCr device index" >&2; exit 2; } [[ ! -e "$OUT" ]] || { echo "refusing to overwrite $OUT" >&2; exit 2; } +if ! [[ "$PREFILL_FIRST_BURST_STEPS" =~ ^[0-9]{1,4}$ ]] || (( 10#$PREFILL_FIRST_BURST_STEPS > 1024 )); then + echo "PREFILL_FIRST_BURST_STEPS must be an integer in range 0..1024" >&2 + exit 2 +fi +if ! [[ "$IDLE_PREFILL_TOKENS" =~ ^[1-9][0-9]{0,4}$ ]] || (( 10#$IDLE_PREFILL_TOKENS > 16384 )); then + echo "IDLE_PREFILL_TOKENS must be an integer in range 1..16384" >&2 + exit 2 +fi ambient_tuning="$(env | grep -E '^(GGML_|DFLASH_|LUCE_|HIP_|ROCR_|HSA_|LD_PRELOAD=|LD_LIBRARY_PATH=)' \ | grep -v '^LUCE_SERVER_BIN=' || true)" if [[ -n "$ambient_tuning" ]]; then @@ -55,17 +84,36 @@ MODEL_SHA256="$(sha256sum "$MODEL" | awk '{print $1}')" IFS=, read -r -a workload_list <<< "$WORKLOADS" IFS=, read -r -a variant_list <<< "$VARIANTS" IFS=, read -r -a client_list <<< "$CLIENTS" -declare -A prompt_offsets=([1]=0 [4]=1 [8]=5 [16]=13) +declare -A prompt_offsets=() +declare -A seen_clients=() +next_prompt_offset=0 +max_clients=0 for c in "${client_list[@]}"; do - [[ -n "${prompt_offsets[$c]+yes}" ]] || { echo "supported CLIENTS are 1,4,8,16" >&2; exit 2; } + [[ "$c" =~ ^[1-9][0-9]*$ ]] || { echo "CLIENTS must contain positive integers" >&2; exit 2; } + [[ -z "${seen_clients[$c]+yes}" ]] || { echo "CLIENTS levels must be distinct" >&2; exit 2; } + seen_clients[$c]=1 + prompt_offsets[$c]="$next_prompt_offset" + next_prompt_offset=$((next_prompt_offset + c)) + if (( c > max_clients )); then + max_clients="$c" + fi done +if [[ -z "$SLOTS" ]]; then + SLOTS=16 + if (( max_clients > SLOTS )); then + SLOTS="$max_clients" + fi +fi +[[ "$SLOTS" =~ ^[1-9][0-9]*$ ]] || { echo "SLOTS must be positive" >&2; exit 2; } +(( SLOTS >= max_clients )) || { echo "SLOTS must be at least the largest CLIENTS level" >&2; exit 2; } for v in "${variant_list[@]}"; do [[ "$v" == luce-k8 || "$v" == luce-k1 || "$v" == llama ]] || { echo "unknown variant $v" >&2; exit 2; } done mkdir -p "$OUT/prompts" for workload in "${workload_list[@]}"; do - python3 "$GENERATOR" --profile "$workload" --out "$OUT/prompts/$workload.jsonl" + python3 "$GENERATOR" --profile "$workload" --clients "$CLIENTS" \ + --out "$OUT/prompts/$workload.jsonl" done server_pid="" @@ -81,7 +129,8 @@ stop_server() { fi server_pid="" } -trap stop_server EXIT INT TERM +trap stop_server EXIT +trap 'exit 130' INT TERM wait_health() { local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) @@ -95,8 +144,9 @@ wait_health() { write_metadata() { local path="$1" variant="$2" workload="$3" clients="$4" repeat="$5" binary="$6" max_prefills="$7" command_file="$8" + local prompt_offset="$9" client_levels="${10}" slots="${11}" rocr_visible_devices="${12}" prefill_first_burst_steps="${13}" expected_gpu_arch="${14}" idle_prefill_tokens="${15}" python3 -c 'import hashlib,json,pathlib,subprocess,sys -p,variant,workload,clients,repeat,binary,max_prefills,cmd_file,model_sha,prompts,repo=sys.argv[1:] +p,variant,workload,clients,repeat,binary,max_prefills,cmd_file,model_sha,prompts,repo,prompt_offset,client_levels,slots,rocr_visible,prefill_first_burst_steps,expected_gpu_arch,idle_prefill_tokens=sys.argv[1:] digest=lambda x: hashlib.sha256(pathlib.Path(x).read_bytes()).hexdigest() libs={} for line in subprocess.run(["ldd",binary],text=True,capture_output=True).stdout.splitlines(): @@ -114,11 +164,18 @@ obj={"variant":variant,"workload":workload,"clients":int(clients),"repeat":int(r "max_concurrent_prefills":int(max_prefills),"server_binary":str(pathlib.Path(binary).resolve()), "server_binary_sha256":digest(binary),"model_sha256":model_sha, "prompt_file_sha256":digest(prompts),"server_command":pathlib.Path(cmd_file).read_text().strip(), + "client_levels":[int(x) for x in client_levels.split(",")], + "prompt_offset":int(prompt_offset),"server_slots":int(slots), + "rocr_visible_devices":rocr_visible, + "expected_gpu_arch":expected_gpu_arch or None, "resolved_shared_library_sha256":libs, + "prefill_first_burst_steps":int(prefill_first_burst_steps) if variant != "llama" else None, + "idle_prefill_tokens":int(idle_prefill_tokens) if variant != "llama" else None, "lucebox_git_head":lucebox_git_head if variant != "llama" else None, "server_version":server_version} pathlib.Path(p).write_text(json.dumps(obj,indent=2,sort_keys=True)+"\n")' \ - "$path" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" "$command_file" "$MODEL_SHA256" "$OUT/prompts/$workload.jsonl" "$REPO" + "$path" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" "$command_file" \ + "$MODEL_SHA256" "$OUT/prompts/$workload.jsonl" "$REPO" "$prompt_offset" "$client_levels" "$slots" "$rocr_visible_devices" "$prefill_first_burst_steps" "$expected_gpu_arch" "$idle_prefill_tokens" } run_case() { @@ -132,15 +189,44 @@ run_case() { capacity=$((SLOTS * max_ctx)) local case_dir="$OUT/$workload/c$clients/r$repeat/$variant" mkdir -p "$case_dir" - local -a command launch_command if [[ "$variant" == llama ]]; then binary="$LLAMA_SERVER_BIN"; model_id=qwen36-llama; max_prefills=0 - command=("$binary" -m "$MODEL" -ngl all --parallel "$SLOTS" -c "$capacity" - -b 2048 -ub 512 --cont-batching --no-context-shift --no-mmap -fa on - -ctk q4_0 -ctv q4_0 --no-cache-prompt --host 127.0.0.1 --port "$PORT" --alias "$model_id") else binary="$LUCE_SERVER_BIN"; model_id=qwen36-luce [[ "$variant" == luce-k8 ]] && max_prefills=8 || max_prefills=1 + fi + if [[ -n "$EXPECTED_GPU_ARCH" ]]; then + if ! command -v rocminfo >/dev/null; then + echo "EXPECTED_GPU_ARCH requires rocminfo" >&2 + return 1 + fi + if ! env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" rocminfo 2>/dev/null | grep -F -- "$EXPECTED_GPU_ARCH" > "$case_dir/gpu-identity.txt"; then + echo "isolated ROCr device did not report expected architecture $EXPECTED_GPU_ARCH" >&2 + return 1 + fi + fi + if [[ "$variant" == llama ]]; then + printf '%q ' env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" "$binary" --list-devices \ + > "$case_dir/llama-list-devices-command.txt" + printf '\n' >> "$case_dir/llama-list-devices-command.txt" + if ! env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" "$binary" --list-devices \ + > "$case_dir/llama-list-devices.txt" 2>&1; then + echo "llama.cpp --list-devices failed under isolated ROCr device $GPU_DEVICE" >&2 + return 1 + fi + if ! grep -Eq '^[[:space:]]*ROCm[0-9]+:[[:space:]]+[^[:space:]]' \ + "$case_dir/llama-list-devices.txt"; then + echo "llama.cpp did not expose a ROCm device under isolated ROCr device $GPU_DEVICE" >&2 + return 1 + fi + fi + local -a command launch_command + if [[ "$variant" == llama ]]; then + command=("$binary" -m "$MODEL" -ngl all -lv 4 --reasoning off --reasoning-format none + --parallel "$SLOTS" -c "$capacity" + -b 2048 -ub 512 --cont-batching --no-context-shift --no-mmap -fa on + -ctk q4_0 -ctv q4_0 --no-cache-prompt --host 127.0.0.1 --port "$PORT" --alias "$model_id") + else command=("$binary" "$MODEL" --target-device hip:0 --paged-attention --max-concurrency "$SLOTS" --kv-pool-tokens "$capacity" --max-ctx "$max_ctx" --cache-type-k q4_0 --cache-type-v q4_0 --fa-window 0 @@ -148,14 +234,18 @@ run_case() { --host 127.0.0.1 --port "$PORT" --model-name "$model_id") fi if [[ "$variant" == llama ]]; then - launch_command=("${command[@]}") + launch_command=(env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" "${command[@]}") else - launch_command=(env DFLASH_IGNORE_EOS=1 + launch_command=(env ROCR_VISIBLE_DEVICES="$GPU_DEVICE" DFLASH_IGNORE_EOS=1 DFLASH_MIN_TOKENS="$WARMUP_TOKENS" + DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS" + DFLASH_IDLE_PREFILL_TOKENS="$IDLE_PREFILL_TOKENS" DFLASH_MAX_CONCURRENT_PREFILLS="$max_prefills" "${command[@]}") fi printf '%q ' "${launch_command[@]}" > "$case_dir/server-command.txt"; printf '\n' >> "$case_dir/server-command.txt" - write_metadata "$case_dir/server-metadata.json" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" "$case_dir/server-command.txt" + local offset="${prompt_offsets[$clients]}" + write_metadata "$case_dir/server-metadata.json" "$variant" "$workload" "$clients" "$repeat" "$binary" "$max_prefills" \ + "$case_dir/server-command.txt" "$offset" "$CLIENTS" "$SLOTS" "$GPU_DEVICE" "$PREFILL_FIRST_BURST_STEPS" "$EXPECTED_GPU_ARCH" "$IDLE_PREFILL_TOKENS" echo "[run] $workload C=$clients repeat=$repeat variant=$variant" "${launch_command[@]}" > "$case_dir/server.log" 2>&1 & @@ -167,7 +257,35 @@ run_case() { return 1 fi - local offset="${prompt_offsets[$clients]}" prompts="$OUT/prompts/$workload.jsonl" + if [[ "$variant" == llama ]]; then + if ! python3 -c 'import pathlib,re,sys +source=pathlib.Path(sys.argv[1]) +matches=[] +for line in source.read_text(encoding="utf-8",errors="replace").splitlines(): + match=re.search(r"\boffloaded ([1-9][0-9]*)/([1-9][0-9]*) layers to GPU\b",line) + if match and match.group(1) == match.group(2): matches.append(line) +if not matches: raise SystemExit(1) +pathlib.Path(sys.argv[2]).write_text("\n".join(matches)+"\n",encoding="utf-8")' \ + "$case_dir/server.log" "$case_dir/server-gpu-proof.txt"; then + echo "llama.cpp server did not report a positive full GPU offload" >&2 + tail -n 80 "$case_dir/server.log" >&2 || true + stop_server + sleep "$COOLDOWN_SECONDS" + return 1 + fi + elif [[ -n "$EXPECTED_GPU_ARCH" ]]; then + if ! grep -F -- "$EXPECTED_GPU_ARCH" "$case_dir/server.log" \ + > "$case_dir/server-gpu-proof.txt"; then + echo "Lucebox server did not report expected GPU architecture $EXPECTED_GPU_ARCH" >&2 + tail -n 80 "$case_dir/server.log" >&2 || true + stop_server + sleep "$COOLDOWN_SECONDS" + return 1 + fi + fi + + + local prompts="$OUT/prompts/$workload.jsonl" local status=0 if ! python3 "$CLIENT" --base-url "http://127.0.0.1:$PORT/v1" --model "$model_id" \ --clients "$clients" --prompt-file "$prompts" --prompt-offset "$offset" \ diff --git a/harness/benchmarks/concurrency/summarize_canonical_concurrency.py b/harness/benchmarks/concurrency/summarize_canonical_concurrency.py deleted file mode 100755 index e250562f0..000000000 --- a/harness/benchmarks/concurrency/summarize_canonical_concurrency.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Summarize canonical concurrent benchmark reports into one comparison table.""" - -from __future__ import annotations - -import argparse -import json -import statistics -from collections import defaultdict -from pathlib import Path -from typing import Any - - -def fmt(value: Any, digits: int = 2) -> str: - return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" - - -def output_signature(report: dict[str, Any]) -> tuple[tuple[str, str, str], ...] | None: - level = report["levels"][0] - rows = [] - for wave in level.get("wave_results", []): - for request in wave.get("requests_detail", []): - try: - rows.append(( - request["case_id"], request["content_sha256"], - request["reasoning_content_sha256"], - )) - except KeyError: - return None - if len(rows) != level["requests"]: - return None - return tuple(rows) - - -def summarize(root: Path) -> str: - GroupKey = tuple[str, int, str, int | None, int] - FamilyKey = tuple[str, int, int | None, int] - groups: dict[GroupKey, list[dict[str, Any]]] = defaultdict(list) - repeat_ids: dict[GroupKey, set[int]] = defaultdict(set) - variant_repeats: dict[FamilyKey, dict[str, set[int]]] = defaultdict(dict) - for path in root.glob("*/c*/r*/*/bench.json"): - report = json.loads(path.read_text(encoding="utf-8")) - level = report["levels"][0] - metadata = report["server_metadata"] - suite = report.get("suite") - variant = metadata.get("variant") - clients = level.get("clients") - requests = level.get("requests") - repeat = metadata.get("repeat") - case_limit = report.get("case_limit") - if not isinstance(suite, str) or not isinstance(variant, str): - raise ValueError(f"invalid suite or variant metadata: {path}") - if isinstance(clients, bool) or not isinstance(clients, int) or clients < 1: - raise ValueError(f"invalid client count: {path}") - if isinstance(requests, bool) or not isinstance(requests, int) or requests < 1: - raise ValueError(f"invalid request count: {path}") - if case_limit is not None and ( - isinstance(case_limit, bool) or not isinstance(case_limit, int) or case_limit < 1 - ): - raise ValueError(f"invalid case_limit: {path}") - if isinstance(repeat, bool) or not isinstance(repeat, int) or repeat < 1: - raise ValueError(f"missing or invalid repeat id: {path}") - if level["failures"] or level["fixed_token_workload_valid"] is not True: - raise ValueError(f"invalid measured report: {path}") - if variant.endswith("ddtree"): - proof = report.get("ddtree_proof") - if not isinstance(proof, dict): - raise ValueError(f"missing positive DDTree proof: {path}") - steps = proof.get("ddtree_steps") - if ( - isinstance(steps, bool) or not isinstance(steps, int) or steps <= 0 - or proof.get("requests_proven") != requests - ): - raise ValueError(f"missing positive DDTree proof: {path}") - key = (suite, clients, variant, case_limit, requests) - if repeat in repeat_ids[key]: - raise ValueError(f"duplicate repeat id {repeat} for {key}") - repeat_ids[key].add(repeat) - groups[key].append(report) - family = (suite, clients, case_limit, requests) - variant_repeats.setdefault(family, {}).setdefault(variant, set()).add(repeat) - for family, variants in variant_repeats.items(): - if len({frozenset(repeats) for repeats in variants.values()}) > 1: - raise ValueError(f"mismatched repeat sets for {family}") - if not groups: - raise ValueError(f"no canonical reports under {root}") - lines = [ - "# Canonical Qwen3.6 concurrency benchmark", "", - "| Suite | C | Cases | Variant | Repeats | Goodput tok/s | Output-window tok/s | " - "Prompt tok/s to first | Request decode tok/s | TTFT median s | TTFT max s | " - "DDTree AL | Acceptance | Stable output |", - "| :--- | ---: | ---: | :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :---: |", - ] - sort_key = lambda item: ( - item[0][0], item[0][1], item[0][2], item[0][3] is not None, - item[0][3] or 0, item[0][4], - ) - for (suite, clients, variant, _case_limit, _requests), reports in sorted( - groups.items(), key=sort_key - ): - levels = [report["levels"][0] for report in reports] - proofs = [report.get("ddtree_proof") for report in reports] - med = lambda key: statistics.median(level[key] for level in levels) - al = ( - statistics.median(proof["mean_accepted_length"] for proof in proofs) - if all(proof is not None for proof in proofs) else None - ) - acceptance = ( - statistics.median(proof["acceptance_rate"] for proof in proofs) - if all(proof is not None for proof in proofs) else None - ) - acceptance_text = f"{100 * acceptance:.1f}%" if acceptance is not None else "n/a" - signatures = [output_signature(report) for report in reports] - complete = len(reports) >= 2 and all(signature is not None for signature in signatures) - stable = "YES" if complete and len(set(signatures)) == 1 else "NO" if complete else "n/a" - lines.append( - f"| {suite} | {clients} | {levels[0]['requests']} | {variant} | {len(reports)} | " - f"{fmt(med('aggregate_tok_s'))} | {fmt(med('output_window_tok_s'))} | " - f"{fmt(med('prompt_tokens_per_s_to_first_token'))} | " - f"{fmt(med('request_decode_tok_s_median'))} | " - f"{fmt(med('ttft_median_s'), 3)} | {fmt(med('ttft_max_s'), 3)} | " - f"{fmt(al)} | {acceptance_text} | {stable} |" - ) - return "\n".join(lines) + "\n" - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("root", type=Path) - parser.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - text = summarize(args.root) - args.out.write_text(text, encoding="utf-8") - print(text, end="") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/harness/benchmarks/concurrency/summarize_concurrency.py b/harness/benchmarks/concurrency/summarize_concurrency.py index 69563a35c..7071b0569 100755 --- a/harness/benchmarks/concurrency/summarize_concurrency.py +++ b/harness/benchmarks/concurrency/summarize_concurrency.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -"""Summarize paired Lucebox/llama.cpp concurrency benchmark reports.""" +"""Summarize ragged or canonical Lucebox concurrency benchmark reports.""" from __future__ import annotations import argparse import json +import math import statistics from collections import defaultdict from pathlib import Path +from typing import Any def load_reports(root: Path) -> list[dict]: @@ -38,6 +40,44 @@ def median(values: list[float]) -> float: return statistics.median(values) +def native_metric_median( + levels: list[dict[str, Any]], key: str, context: str, +) -> float | None: + """Require native telemetry on every repeat or on none of them.""" + available: list[bool] = [] + values: list[float] = [] + for level in levels: + token_count_complete = ( + level.get("server_native_prefill_token_count_complete") is True + ) + timing_complete = ( + level.get("server_native_prefill_timing_complete") is True + ) + if token_count_complete != timing_complete: + raise ValueError( + f"{context}: mismatched native prefill completeness flags" + ) + complete = token_count_complete and timing_complete + value = level.get(key) + if complete: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError(f"{context}: invalid complete native metric {key}") + values.append(float(value)) + elif value is not None: + raise ValueError( + f"{context}: native metric {key} present without complete telemetry" + ) + available.append(complete) + if any(available) and not all(available): + raise ValueError(f"{context}: partial native prefill telemetry across repeats") + return median(values) if values else None + + def paired_delta( grouped: dict[tuple[str, int, str], list[dict]], workload: str, @@ -99,12 +139,16 @@ def summarize(reports: list[dict]) -> str: "Aggregate output goodput includes queueing, prefill, and decode. " "Output-window goodput starts at the first observed output and is decode-facing, " "but it can include staggered prefill. Prompt tok/s to first token includes " - "admission and TTFT.", "", + "admission and TTFT. Native prefill metrics come only from terminal " + "usage.timings; they remain n/a when a server does not expose those fields. " + "Native prefill tok/s divides total prefilled_tokens by the common-origin " + "prefill window, so overlapping request times are not summed.", "", "| Workload | C | Variant | Repeats | Output goodput tok/s | " "Output-window tok/s | Request decode tok/s | Prompt tok/s to first | " - "TTFT median s | TTFT max s | Stable output | vs llama | Decode vs llama | K8 vs K1 |", + "TTFT median s | TTFT max s | Stable output | vs llama | Decode vs llama | " + "K8 vs K1 | Native prefill tok/s | Native prefill window ms |", "| :--- | ---: | :--- | ---: | ---: | ---: | ---: | ---: | ---: | " - "---: | :---: | ---: | ---: | ---: |", + "---: | :---: | ---: | ---: | ---: | ---: | ---: |", ] for workload, clients, variant in sorted(grouped): items = grouped[(workload, clients, variant)] @@ -127,6 +171,14 @@ def summarize(reports: list[dict]) -> str: if item["level"].get("prompt_tokens_per_s_to_first_token") is not None ] prompt_rate = median(prompt_rate_values) if prompt_rate_values else None + native_levels = [item["level"] for item in items] + native_context = f"{workload} C={clients} {variant}" + native_prefill_rate = native_metric_median( + native_levels, "server_native_prefill_tokens_per_s", native_context, + ) + native_prefill_window_ms = native_metric_median( + native_levels, "server_native_prefill_window_ms", native_context, + ) ttft_median = median([item["level"]["ttft_median_s"] for item in items]) ttft_max = median([item["level"]["ttft_max_s"] for item in items]) output_hashes = { @@ -162,25 +214,181 @@ def summarize(reports: list[dict]) -> str: output_window_text = f"{output_window:.2f}" if output_window is not None else "n/a" request_decode_text = f"{request_decode:.2f}" if request_decode is not None else "n/a" prompt_rate_text = f"{prompt_rate:.2f}" if prompt_rate is not None else "n/a" + native_rate_text = ( + f"{native_prefill_rate:.2f}" if native_prefill_rate is not None else "n/a" + ) + native_ms_text = ( + f"{native_prefill_window_ms:.1f}" + if native_prefill_window_ms is not None else "n/a" + ) lines.append( f"| {workload} | {clients} | {variant} | {len(items)} | {goodput:.2f} | " f"{output_window_text} | {request_decode_text} | {prompt_rate_text} | " f"{ttft_median:.3f} | {ttft_max:.3f} | {stable} | {vs_llama} | " - f"{decode_vs_llama} | {vs_k1} |" + f"{decode_vs_llama} | {vs_k1} | {native_rate_text} | {native_ms_text} |" ) lines.append("") return "\n".join(lines) +def fmt(value: Any, digits: int = 2) -> str: + return f"{value:.{digits}f}" if isinstance(value, (int, float)) else "n/a" + + +def output_signature(report: dict[str, Any]) -> tuple[tuple[str, str, str], ...] | None: + level = report["levels"][0] + rows = [] + for wave in level.get("wave_results", []): + for request in wave.get("requests_detail", []): + try: + rows.append(( + request["case_id"], request["content_sha256"], + request["reasoning_content_sha256"], + )) + except KeyError: + return None + if len(rows) != level["requests"]: + return None + return tuple(rows) + + +def summarize_canonical(root: Path) -> str: + GroupKey = tuple[str, int, str, int | None, int] + FamilyKey = tuple[str, int, int | None, int] + PromptFamilyKey = tuple[str, int, int | None] + groups: dict[GroupKey, list[dict[str, Any]]] = defaultdict(list) + repeat_ids: dict[GroupKey, set[int]] = defaultdict(set) + variant_repeats: dict[FamilyKey, dict[str, set[int]]] = defaultdict(dict) + prompt_hashes: dict[PromptFamilyKey, str] = {} + for path in root.glob("*/c*/r*/*/bench.json"): + report = json.loads(path.read_text(encoding="utf-8")) + levels = report.get("levels") + if not isinstance(levels, list) or len(levels) != 1: + raise ValueError(f"{path}: expected exactly one client level") + level = levels[0] + metadata = report["server_metadata"] + suite = report.get("suite") + variant = metadata.get("variant") + clients = level.get("clients") + requests = level.get("requests") + repeat = metadata.get("repeat") + case_limit = report.get("case_limit") + if not isinstance(suite, str) or not isinstance(variant, str): + raise ValueError(f"invalid suite or variant metadata: {path}") + if isinstance(clients, bool) or not isinstance(clients, int) or clients < 1: + raise ValueError(f"invalid client count: {path}") + if isinstance(requests, bool) or not isinstance(requests, int) or requests < 1: + raise ValueError(f"invalid request count: {path}") + if case_limit is not None and ( + isinstance(case_limit, bool) or not isinstance(case_limit, int) or case_limit < 1 + ): + raise ValueError(f"invalid case_limit: {path}") + if isinstance(repeat, bool) or not isinstance(repeat, int) or repeat < 1: + raise ValueError(f"missing or invalid repeat id: {path}") + if ( + level["failures"] + or level["fixed_token_workload_valid"] is not True + or level.get("token_count_complete") is not True + or level.get("prompt_token_count_complete") is not True + ): + raise ValueError(f"failed or incomplete token accounting: {path}") + prompt_hash = report.get("prompt_file_sha256") + if not isinstance(prompt_hash, str) or not prompt_hash: + raise ValueError(f"missing prompt_file_sha256: {path}") + prompt_family = (suite, clients, case_limit) + previous_hash = prompt_hashes.setdefault(prompt_family, prompt_hash) + if previous_hash != prompt_hash: + raise ValueError( + f"{suite} C={clients} case_limit={case_limit}: prompt files differ" + ) + if variant.endswith("ddtree"): + proof = report.get("ddtree_proof") + if not isinstance(proof, dict): + raise ValueError(f"missing positive DDTree proof: {path}") + steps = proof.get("ddtree_steps") + if ( + isinstance(steps, bool) or not isinstance(steps, int) or steps <= 0 + or proof.get("requests_proven") != requests + ): + raise ValueError(f"missing positive DDTree proof: {path}") + key = (suite, clients, variant, case_limit, requests) + if repeat in repeat_ids[key]: + raise ValueError(f"duplicate repeat id {repeat} for {key}") + repeat_ids[key].add(repeat) + groups[key].append(report) + family = (suite, clients, case_limit, requests) + variant_repeats.setdefault(family, {}).setdefault(variant, set()).add(repeat) + for family, variants in variant_repeats.items(): + if len({frozenset(repeats) for repeats in variants.values()}) > 1: + raise ValueError(f"mismatched repeat sets for {family}") + if not groups: + raise ValueError(f"no canonical reports under {root}") + lines = [ + "# Canonical Qwen3.6 concurrency benchmark", "", + "| Suite | C | Cases | Variant | Repeats | Goodput tok/s | Output-window tok/s | " + "Prompt tok/s to first | Request decode tok/s | TTFT median s | TTFT max s | " + "DDTree AL | Acceptance | Stable output | Native prefill tok/s | " + "Native prefill window ms |", + "| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | " + "---: | ---: | ---: | :---: | ---: | ---: |", + ] + sort_key = lambda item: ( + item[0][0], item[0][1], item[0][2], item[0][3] is not None, + item[0][3] or 0, item[0][4], + ) + for (suite, clients, variant, _case_limit, _requests), reports in sorted( + groups.items(), key=sort_key + ): + levels = [report["levels"][0] for report in reports] + proofs = [report.get("ddtree_proof") for report in reports] + med = lambda key, current_levels=levels: statistics.median( + level[key] for level in current_levels + ) + native_context = f"{suite} C={clients} {variant} canonical" + native_prefill_rate = native_metric_median( + levels, "server_native_prefill_tokens_per_s", native_context, + ) + native_prefill_window_ms = native_metric_median( + levels, "server_native_prefill_window_ms", native_context, + ) + al = ( + statistics.median(proof["mean_accepted_length"] for proof in proofs) + if all(proof is not None for proof in proofs) else None + ) + acceptance = ( + statistics.median(proof["acceptance_rate"] for proof in proofs) + if all(proof is not None for proof in proofs) else None + ) + acceptance_text = f"{100 * acceptance:.1f}%" if acceptance is not None else "n/a" + signatures = [output_signature(report) for report in reports] + complete = len(reports) >= 2 and all(signature is not None for signature in signatures) + stable = "YES" if complete and len(set(signatures)) == 1 else "NO" if complete else "n/a" + lines.append( + f"| {suite} | {clients} | {levels[0]['requests']} | {variant} | {len(reports)} | " + f"{fmt(med('aggregate_tok_s'))} | {fmt(med('output_window_tok_s'))} | " + f"{fmt(med('prompt_tokens_per_s_to_first_token'))} | " + f"{fmt(med('request_decode_tok_s_median'))} | " + f"{fmt(med('ttft_median_s'), 3)} | {fmt(med('ttft_max_s'), 3)} | " + f"{fmt(al)} | {acceptance_text} | {stable} | " + f"{fmt(native_prefill_rate)} | {fmt(native_prefill_window_ms, 1)} |" + ) + return "\n".join(lines) + "\n" + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("root", type=Path) parser.add_argument("--out", type=Path) + parser.add_argument("--format", choices=("ragged", "canonical"), default="ragged") args = parser.parse_args() - text = summarize(load_reports(args.root)) + text = ( + summarize_canonical(args.root) + if args.format == "canonical" + else summarize(load_reports(args.root)) + ) if args.out: - args.out.write_text(text + "\n", encoding="utf-8") - print(text) + args.out.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8") + print(text, end="" if text.endswith("\n") else "\n") return 0 diff --git a/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py b/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py deleted file mode 100644 index 602975031..000000000 --- a/harness/benchmarks/concurrency/test_attach_ddtree_metrics.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for attach_ddtree_metrics.py.""" - -from __future__ import annotations - -import importlib.util -import json -import tempfile -import unittest -from pathlib import Path - -SCRIPT = Path(__file__).with_name("attach_ddtree_metrics.py") -SPEC = importlib.util.spec_from_file_location("attach_ddtree_metrics", SCRIPT) -assert SPEC is not None and SPEC.loader is not None -proof = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(proof) - - -class DDTreeProofTests(unittest.TestCase): - def test_attaches_acceptance_from_matched_requests(self) -> None: - report = {"levels": [{"wave_results": [{"requests_detail": [ - {"response_id": "a", "error": None}, - {"response_id": "b", "error": None}, - ]}]}]} - metrics = { - "a": {"ddtree_steps": 2, "ddtree_accepted_tokens": 8, "target_forwards": 4}, - "b": {"ddtree_steps": 3, "ddtree_accepted_tokens": 12, "target_forwards": 6}, - } - proof.attach(report, metrics) - self.assertEqual(report["ddtree_proof"]["ddtree_steps"], 5) - self.assertEqual(report["ddtree_proof"]["mean_accepted_length"], 5.0) - self.assertEqual(report["ddtree_proof"]["acceptance_rate"], 5 / 16) - - def test_missing_or_zero_step_proof_fails_closed(self) -> None: - report = {"levels": [{"wave_results": [{"requests_detail": [ - {"response_id": "a", "error": None}, - ]}]}]} - with self.assertRaisesRegex(ValueError, "missing concurrency metric"): - proof.attach(report, {}) - metrics = {"a": { - "ddtree_steps": 0, "ddtree_accepted_tokens": 0, "target_forwards": 1, - }} - with self.assertRaisesRegex(ValueError, "ddtree_steps must be positive"): - proof.attach(report, metrics) - - - def test_boolean_counters_are_not_integers(self) -> None: - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "server.log" - path.write_text("[concurrency-metrics] " + json.dumps({ - "response_id": "a", "ddtree_steps": True, - "ddtree_accepted_tokens": 1, "target_forwards": 1, - }) + "\n", encoding="utf-8") - with self.assertRaisesRegex(ValueError, "invalid ddtree_steps"): - proof.load_metrics(path) - - -if __name__ == "__main__": - unittest.main() diff --git a/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py b/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py index 5c7006e61..cf315d5f2 100644 --- a/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/test_canonical_concurrent_benchmark.py @@ -80,6 +80,42 @@ def fake_level(clients, args, prompts, offset): self.assertEqual(report["levels"][0]["waves"], 2) self.assertEqual(report["levels"][0]["requests"], 10) + def test_aggregate_waves_sums_native_prefill_windows(self) -> None: + def wave(tokens: int, window_ms: float) -> dict: + return { + "requests_detail": [{ + "error": None, "completion_tokens": 8, "prompt_tokens": 4, + "request_decode_tok_s": 2.0, "ttft_s": 0.1, + }], + "failures": 0, "wall_s": 1.0, "output_window_s": 0.8, + "prompt_to_first_token_s": 0.2, + "server_native_prefilled_tokens_total": tokens, + "server_native_prefill_window_ms": window_ms, + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": True, + } + + level = benchmark.aggregate_waves(1, [wave(40, 100.0), wave(60, 300.0)]) + self.assertEqual(level["server_native_prefilled_tokens_total"], 100) + self.assertEqual(level["server_native_prefill_window_ms"], 400.0) + self.assertEqual(level["server_native_prefill_tokens_per_s"], 250.0) + self.assertTrue(level["server_native_prefill_token_count_complete"]) + self.assertTrue(level["server_native_prefill_timing_complete"]) + + incomplete = wave(60, 300.0) + incomplete.update({ + "server_native_prefilled_tokens_total": None, + "server_native_prefill_window_ms": None, + "server_native_prefill_token_count_complete": False, + "server_native_prefill_timing_complete": False, + }) + level = benchmark.aggregate_waves(1, [wave(40, 100.0), incomplete]) + self.assertIsNone(level["server_native_prefilled_tokens_total"]) + self.assertIsNone(level["server_native_prefill_window_ms"]) + self.assertIsNone(level["server_native_prefill_tokens_per_s"]) + self.assertFalse(level["server_native_prefill_token_count_complete"]) + self.assertFalse(level["server_native_prefill_timing_complete"]) + def test_rejects_partial_tail_wave(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -140,9 +176,45 @@ def test_retirement_wait_matches_every_response(self) -> None: with self.assertRaisesRegex(TimeoutError, "did not retire"): benchmark.wait_for_retirement(log, ["missing"], 0.01) + def test_attaches_ddtree_proof_from_matched_requests(self) -> None: + report = {"levels": [{"wave_results": [{"requests_detail": [ + {"response_id": "a", "error": None}, + {"response_id": "b", "error": None}, + ]}]}]} + metrics = { + "a": {"ddtree_steps": 2, "ddtree_accepted_tokens": 8, "target_forwards": 4}, + "b": {"ddtree_steps": 3, "ddtree_accepted_tokens": 12, "target_forwards": 6}, + } + benchmark.attach_ddtree_proof(report, metrics) + self.assertEqual(report["ddtree_proof"]["ddtree_steps"], 5) + self.assertEqual(report["ddtree_proof"]["mean_accepted_length"], 5.0) + self.assertEqual(report["ddtree_proof"]["acceptance_rate"], 5 / 16) + + def test_missing_or_zero_step_ddtree_proof_fails_closed(self) -> None: + report = {"levels": [{"wave_results": [{"requests_detail": [ + {"response_id": "a", "error": None}, + ]}]}]} + with self.assertRaisesRegex(ValueError, "missing concurrency metric"): + benchmark.attach_ddtree_proof(report, {}) + metrics = {"a": { + "ddtree_steps": 0, "ddtree_accepted_tokens": 0, "target_forwards": 1, + }} + with self.assertRaisesRegex(ValueError, "ddtree_steps must be positive"): + benchmark.attach_ddtree_proof(report, metrics) + + def test_boolean_ddtree_counters_are_not_integers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "server.log" + path.write_text("[concurrency-metrics] " + json.dumps({ + "response_id": "a", "ddtree_steps": True, + "ddtree_accepted_tokens": 1, "target_forwards": 1, + }) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid ddtree_steps"): + benchmark.load_ddtree_metrics(path) + def test_blog_generator_matches_bench_he_source(self) -> None: generator_spec = importlib.util.spec_from_file_location( - "generate_blog_prompts", HERE / "generate_blog_prompts.py" + "generate_prompts", HERE / "generate_prompts.py" ) assert generator_spec is not None and generator_spec.loader is not None generator = importlib.util.module_from_spec(generator_spec) @@ -162,7 +234,7 @@ def test_blog_generator_matches_bench_he_source(self) -> None: def test_summary_keeps_suites_separate_and_reports_acceptance(self) -> None: summary_spec = importlib.util.spec_from_file_location( - "summarize_canonical", HERE / "summarize_canonical_concurrency.py" + "summarize_concurrency", HERE / "summarize_concurrency.py" ) assert summary_spec is not None and summary_spec.loader is not None summary = importlib.util.module_from_spec(summary_spec) @@ -174,9 +246,11 @@ def test_summary_keeps_suites_separate_and_reports_acceptance(self) -> None: path.parent.mkdir(parents=True) report = { "suite": suite, "case_limit": None, + "prompt_file_sha256": f"{suite}-prompts", "server_metadata": {"variant": variant, "repeat": 1}, "levels": [{ "clients": 1, "failures": 0, "fixed_token_workload_valid": True, + "token_count_complete": True, "prompt_token_count_complete": True, "requests": 1, "aggregate_tok_s": 10.0, "output_window_tok_s": 11.0, "prompt_tokens_per_s_to_first_token": 13.0, @@ -184,20 +258,67 @@ def test_summary_keeps_suites_separate_and_reports_acceptance(self) -> None: "ttft_median_s": 0.1, "ttft_max_s": 0.2, }], } + if suite == "he-raw": + report["levels"][0].update({ + "server_native_prefill_window_ms": 400.0, + "server_native_prefill_tokens_per_s": 250.0, + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": True, + }) if acceptance is not None: report["ddtree_proof"] = { "ddtree_steps": 1, "requests_proven": 1, "mean_accepted_length": 5.6, "acceptance_rate": acceptance, } path.write_text(json.dumps(report), encoding="utf-8") - text = summary.summarize(root) + text = summary.summarize_canonical(root) self.assertIn("| he-raw | 1 | 1 | blog-ddtree", text) self.assertIn("5.60 | 35.0%", text) self.assertIn("| gsm | 1 | 1 | ar", text) + self.assertIn("Native prefill tok/s | Native prefill window ms", text) + he_row = next(line for line in text.splitlines() if "| he-raw |" in line) + gsm_row = next(line for line in text.splitlines() if "| gsm |" in line) + self.assertEqual(he_row.split("|")[-3].strip(), "250.00") + self.assertEqual(he_row.split("|")[-2].strip(), "400.0") + self.assertEqual(gsm_row.split("|")[-3].strip(), "n/a") + self.assertEqual(gsm_row.split("|")[-2].strip(), "n/a") + + def test_canonical_summary_rejects_mismatched_native_completeness(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_concurrency_native_flags", HERE / "summarize_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "gsm" / "c1" / "r1" / "ar" / "bench.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({ + "suite": "gsm", "case_limit": None, + "prompt_file_sha256": "gsm-prompts", + "server_metadata": {"variant": "ar", "repeat": 1}, + "levels": [{ + "clients": 1, "requests": 1, "failures": 0, + "fixed_token_workload_valid": True, + "token_count_complete": True, + "prompt_token_count_complete": True, + "aggregate_tok_s": 10.0, + "output_window_tok_s": 11.0, + "prompt_tokens_per_s_to_first_token": 13.0, + "request_decode_tok_s_median": 12.0, + "ttft_median_s": 0.1, + "ttft_max_s": 0.2, + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": False, + }], + }), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "mismatched native prefill"): + summary.summarize_canonical(root) def test_summary_reports_output_stability_by_case_id(self) -> None: summary_spec = importlib.util.spec_from_file_location( - "summarize_canonical_stability", HERE / "summarize_canonical_concurrency.py" + "summarize_concurrency_stability", HERE / "summarize_concurrency.py" ) assert summary_spec is not None and summary_spec.loader is not None summary = importlib.util.module_from_spec(summary_spec) @@ -209,10 +330,12 @@ def test_summary_reports_output_stability_by_case_id(self) -> None: path.parent.mkdir(parents=True) path.write_text(json.dumps({ "suite": "gsm", "case_limit": None, + "prompt_file_sha256": "gsm-prompts", "server_metadata": {"variant": "ar", "repeat": repeat}, "levels": [{ "clients": 1, "requests": 1, "failures": 0, "fixed_token_workload_valid": True, "aggregate_tok_s": 10.0, + "token_count_complete": True, "prompt_token_count_complete": True, "output_window_tok_s": 11.0, "prompt_tokens_per_s_to_first_token": 13.0, "request_decode_tok_s_median": 12.0, @@ -223,9 +346,136 @@ def test_summary_reports_output_stability_by_case_id(self) -> None: }]}], }], }), encoding="utf-8") - text = summary.summarize(root) + text = summary.summarize_canonical(root) self.assertIn("| n/a | n/a | NO |", text) + def test_ddtree_proof_rejects_failed_request_before_attachment(self) -> None: + request = {"response_id": "a", "error": "request failed"} + report = {"levels": [{ + "failures": 1, + "wave_results": [{"requests_detail": [request]}], + }]} + metrics = { + "a": {"ddtree_steps": 2, "ddtree_accepted_tokens": 8, "target_forwards": 4}, + } + with self.assertRaisesRegex(ValueError, "failed requests"): + benchmark.attach_ddtree_proof(report, metrics) + self.assertNotIn("ddtree_metrics", request) + self.assertNotIn("ddtree_proof", report) + + def test_ddtree_run_writes_failed_report_without_proof(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + prompt_file = root / "cases.jsonl" + prompt_file.write_text(json.dumps({"id": "p0", "prompt": "prompt"}) + "\n", + encoding="utf-8") + args = argparse.Namespace( + clients=1, prompt_file=prompt_file, case_limit=None, suite="gsm", + ignore_eos=True, server_metadata_json=None, label="ddtree", + base_url="x", model="m", max_tokens=8, temperature=0.0, seed=1, + out=root / "report.json", retire_log=root / "server.log", + ddtree_proof=True, + ) + wave = { + "requests_detail": [{ + "error": "request failed", "completion_tokens": None, + "prompt_tokens": None, "request_decode_tok_s": None, + "ttft_s": None, + }], + "failures": 1, "wall_s": 1.0, "output_window_s": None, + "fixed_token_workload_valid": False, + } + with mock.patch.object(benchmark.base, "run_level", return_value=wave): + self.assertEqual(benchmark.run(args), 1) + report = json.loads(args.out.read_text(encoding="utf-8")) + self.assertNotIn("ddtree_proof", report) + self.assertEqual(report["levels"][0]["failures"], 1) + + def test_canonical_rejects_multiple_client_levels(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_concurrency_levels", HERE / "summarize_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "gsm" / "c1" / "r1" / "ar" / "bench.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({ + "suite": "gsm", "case_limit": None, + "server_metadata": {"variant": "ar", "repeat": 1}, + "levels": [{"clients": 1}, {"clients": 2}], + }), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "exactly one client level"): + summary.summarize_canonical(root) + + def test_canonical_rejects_incomplete_token_accounting(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_concurrency_tokens", HERE / "summarize_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "gsm" / "c1" / "r1" / "ar" / "bench.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({ + "suite": "gsm", "case_limit": None, + "server_metadata": {"variant": "ar", "repeat": 1}, + "levels": [{ + "clients": 1, "requests": 1, "failures": 0, + "fixed_token_workload_valid": True, + "token_count_complete": True, + "prompt_token_count_complete": False, + }], + }), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "incomplete token accounting"): + summary.summarize_canonical(root) + + def test_canonical_rejects_missing_or_mismatched_prompt_file_hash(self) -> None: + summary_spec = importlib.util.spec_from_file_location( + "summarize_concurrency_prompt_hash", HERE / "summarize_concurrency.py" + ) + assert summary_spec is not None and summary_spec.loader is not None + summary = importlib.util.module_from_spec(summary_spec) + summary_spec.loader.exec_module(summary) + + def write_report(root: Path, repeat: int, prompt_hash: str | None) -> None: + path = root / "gsm" / "c1" / f"r{repeat}" / "ar" / "bench.json" + path.parent.mkdir(parents=True) + report = { + "suite": "gsm", "case_limit": None, + "server_metadata": {"variant": "ar", "repeat": repeat}, + "levels": [{ + "clients": 1, "requests": 1, "failures": 0, + "fixed_token_workload_valid": True, + "token_count_complete": True, + "prompt_token_count_complete": True, + "aggregate_tok_s": 1.0, "output_window_tok_s": 1.0, + "prompt_tokens_per_s_to_first_token": 1.0, + "request_decode_tok_s_median": 1.0, + "ttft_median_s": 1.0, "ttft_max_s": 1.0, + }], + } + if prompt_hash is not None: + report["prompt_file_sha256"] = prompt_hash + path.write_text(json.dumps(report), encoding="utf-8") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_report(root, 1, None) + with self.assertRaisesRegex(ValueError, "missing prompt_file_sha256"): + summary.summarize_canonical(root) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_report(root, 1, "prompt-a") + write_report(root, 2, "prompt-b") + with self.assertRaisesRegex(ValueError, "prompt files differ"): + summary.summarize_canonical(root) + if __name__ == "__main__": unittest.main() diff --git a/harness/benchmarks/concurrency/test_canonical_runner_policy.py b/harness/benchmarks/concurrency/test_canonical_runner_policy.py new file mode 100644 index 000000000..d48b49ff0 --- /dev/null +++ b/harness/benchmarks/concurrency/test_canonical_runner_policy.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Focused policy/provenance checks for the canonical concurrency runner.""" + +from __future__ import annotations + +import os +import re +import subprocess +import unittest +from pathlib import Path + +RUNNER = Path(__file__).with_name("run_qwen36_canonical_concurrency.sh") + + +class CanonicalRunnerPolicyTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.script = RUNNER.read_text(encoding="utf-8") + + def test_prefill_first_policy_defaults_off_and_rejects_out_of_range_values(self) -> None: + self.assertIn( + 'PREFILL_FIRST_BURST_STEPS="${PREFILL_FIRST_BURST_STEPS:-0}"', + self.script, + ) + + for value in ( + "-1", + "1025", + "18446744073709551616", + "999999999999999999999999999999999999", + ): + env = os.environ.copy() + env.update( + MODEL="/dev/null", + SERVER_BIN="/bin/true", + PREFILL_FIRST_BURST_STEPS=value, + ) + result = subprocess.run( + [str(RUNNER)], env=env, text=True, capture_output=True, check=False + ) + with self.subTest(value=value): + self.assertEqual(result.returncode, 2) + self.assertIn( + "PREFILL_FIRST_BURST_STEPS must be an integer in range 0..1024", + result.stderr, + ) + self.assertIn( + '(( 10#$PREFILL_FIRST_BURST_STEPS > 1024 ))', + self.script, + ) + + def test_idle_prefill_budget_rejects_zero_and_above_effective_cap(self) -> None: + self.assertIn( + 'IDLE_PREFILL_TOKENS="${IDLE_PREFILL_TOKENS:-4096}"', + self.script, + ) + for value in ( + "0", + "16385", + "18446744073709551616", + "999999999999999999999999999999999999", + ): + env = os.environ.copy() + env.update( + MODEL="/dev/null", + SERVER_BIN="/bin/true", + IDLE_PREFILL_TOKENS=value, + ) + result = subprocess.run( + [str(RUNNER)], env=env, text=True, capture_output=True, check=False + ) + with self.subTest(value=value): + self.assertEqual(result.returncode, 2) + self.assertIn( + "IDLE_PREFILL_TOKENS must be an integer in range 1..16384", + result.stderr, + ) + self.assertIn( + '(( 10#$IDLE_PREFILL_TOKENS > 16384 ))', + self.script, + ) + + def test_policy_is_inside_ar_and_ddtree_launch_arrays_before_command(self) -> None: + launch_blocks = re.findall( + r'launch=\(env (?P.*?)"\$\{command\[@\]\}"\)', + self.script, + flags=re.DOTALL, + ) + self.assertEqual(len(launch_blocks), 2) + self.assertEqual( + sum("DFLASH_DDTREE_ADAPTIVE" in block for block in launch_blocks), 1 + ) + assignment = ( + 'DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS"' + ) + idle_assignment = ( + 'DFLASH_IDLE_PREFILL_TOKENS="$IDLE_PREFILL_TOKENS"' + ) + for block in launch_blocks: + with self.subTest(ddtree="DFLASH_DDTREE_ADAPTIVE" in block): + self.assertIn('ROCR_VISIBLE_DEVICES="$GPU_DEVICE"', block) + self.assertIn(assignment, block) + self.assertIn(idle_assignment, block) + self.assertLess(block.index(assignment), block.index("stdbuf")) + self.assertLess(block.index(idle_assignment), block.index("stdbuf")) + + def test_metadata_records_the_exact_resolved_policy_value(self) -> None: + self.assertIn( + "slots,prefill_first_burst_steps,expected_gpu_arch," + "idle_prefill_tokens=sys.argv[1:]", + self.script, + ) + self.assertIn( + '"prefill_first_burst_steps":int(prefill_first_burst_steps)', + self.script, + ) + self.assertIn( + '"$GPU_DEVICE" "$SLOTS" "$PREFILL_FIRST_BURST_STEPS"', + self.script, + ) + + self.assertIn('"idle_prefill_tokens":int(idle_prefill_tokens)', self.script) + self.assertIn( + '"$EXPECTED_GPU_ARCH" "$IDLE_PREFILL_TOKENS"', self.script + ) + + def test_expected_arch_is_optional_recorded_and_checked_after_health(self) -> None: + self.assertIn('EXPECTED_GPU_ARCH="${EXPECTED_GPU_ARCH:-}"', self.script) + self.assertIn( + '[[ -z "$EXPECTED_GPU_ARCH" || "$EXPECTED_GPU_ARCH" =~ ' + '^gfx[0-9a-f]+$ ]]', + self.script, + ) + self.assertIn('"expected_gpu_arch":expected_gpu_arch or None', self.script) + self.assertIn( + '"$PREFILL_FIRST_BURST_STEPS" "$EXPECTED_GPU_ARCH"', self.script + ) + health = self.script.index("if ! wait_health") + literal_check = self.script.index( + 'grep -F -- "$EXPECTED_GPU_ARCH" "$case_dir/server.log"', health + ) + client = self.script.index("client_common=(", literal_check) + self.assertLess(health, literal_check) + self.assertLess(literal_check, client) + guard = self.script[literal_check:client] + self.assertIn('> "$case_dir/gpu-identity.txt"', guard) + self.assertLess(guard.index("stop_server"), guard.index("return 1")) + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/benchmarks/concurrency/test_concurrency_tools.py b/harness/benchmarks/concurrency/test_concurrency_tools.py index 925694d29..565884a76 100644 --- a/harness/benchmarks/concurrency/test_concurrency_tools.py +++ b/harness/benchmarks/concurrency/test_concurrency_tools.py @@ -5,6 +5,8 @@ import importlib.util import json +import os +import subprocess import tempfile import unittest from pathlib import Path @@ -21,33 +23,174 @@ def load(name: str): return module -generator = load("generate_ragged_prompts") +generator = load("generate_prompts") summarizer = load("summarize_concurrency") class PromptGeneratorTests(unittest.TestCase): def test_cohorts_are_disjoint_ragged_and_mean_matched(self) -> None: records = generator.build_records("short") - self.assertEqual(len(records), 29) + self.assertEqual(len(records), 30) self.assertEqual( [row["cohort"] for row in records], - ["c1"] + ["c4"] * 4 + ["c8"] * 8 + ["c16"] * 16, + ["c2"] * 2 + ["c4"] * 4 + ["c8"] * 8 + ["c16"] * 16, ) - self.assertEqual(len({row["prompt"] for row in records}), 29) + self.assertEqual(len({row["prompt"] for row in records}), 30) by_cohort = { cohort: [row for row in records if row["cohort"] == cohort] - for cohort in ("c1", "c4", "c8", "c16") + for cohort in ("c2", "c4", "c8", "c16") } means = { cohort: sum(row["target_words"] for row in rows) / len(rows) for cohort, rows in by_cohort.items() } self.assertEqual(len(set(means.values())), 1) + self.assertEqual( + {cohort: rows[0]["cohort_offset"] for cohort, rows in by_cohort.items()}, + {"c2": 0, "c4": 2, "c8": 6, "c16": 14}, + ) + self.assertEqual( + {row["target_words"] for row in by_cohort["c2"]}, {250, 550} + ) for cohort in ("c4", "c8", "c16"): self.assertEqual(len({row["target_words"] for row in by_cohort[cohort]}), 4) for row in records: self.assertEqual(len(row["prompt"].split()), row["target_words"]) + def test_extended_matrix_is_deterministic_without_changing_existing_cohorts(self) -> None: + base = generator.build_records("long", (2, 4, 8, 16)) + extended = generator.build_records("long", (2, 4, 8, 16, 32, 33)) + self.assertEqual(base, extended[:len(base)]) + self.assertEqual(len(extended), 95) + self.assertEqual(len({row["id"] for row in extended}), len(extended)) + self.assertEqual(len({row["prompt"] for row in extended}), len(extended)) + offset = 0 + for clients in (2, 4, 8, 16, 32, 33): + cohort = [ + row for row in extended if row["cohort_clients"] == clients + ] + self.assertEqual(len(cohort), clients) + self.assertTrue(all(row["cohort_offset"] == offset for row in cohort)) + self.assertEqual( + [row["cohort_index"] for row in cohort], list(range(clients)) + ) + self.assertEqual( + sum(row["target_words"] for row in cohort), clients * 3000 + ) + offset += clients + + def test_client_level_parser_rejects_reuse(self) -> None: + self.assertEqual(generator.parse_client_levels("2,4,8,16,32"), (2, 4, 8, 16, 32)) + with self.assertRaisesRegex(ValueError, "distinct"): + generator.parse_client_levels("2,4,2") + with self.assertRaisesRegex(ValueError, "positive"): + generator.parse_client_levels("2,0") + + +class RunnerTests(unittest.TestCase): + def test_runners_isolate_and_record_selected_gpu(self) -> None: + for script_name in ( + "run_qwen36_concurrency.sh", + "run_qwen36_canonical_concurrency.sh", + ): + text = (HERE / script_name).read_text(encoding="utf-8") + self.assertIn('GPU_DEVICE="${GPU_DEVICE:-0}"', text) + self.assertIn('ROCR_VISIBLE_DEVICES="$GPU_DEVICE"', text) + self.assertIn('"rocr_visible_devices"', text) + + def test_ragged_runner_derives_offsets_from_requested_matrix(self) -> None: + text = (HERE / "run_qwen36_concurrency.sh").read_text(encoding="utf-8") + self.assertIn('CLIENTS="${CLIENTS:-2,4,8,16}"', text) + self.assertIn('prompt_offsets[$c]="$next_prompt_offset"', text) + self.assertIn('--clients "$CLIENTS"', text) + self.assertNotIn("prompt_offsets=([", text) + + def test_ragged_runner_records_prefill_first_policy(self) -> None: + text = (HERE / "run_qwen36_concurrency.sh").read_text(encoding="utf-8") + self.assertIn('PREFILL_FIRST_BURST_STEPS="${PREFILL_FIRST_BURST_STEPS:-0}"', text) + self.assertIn('DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS"', text) + self.assertIn('"prefill_first_burst_steps"', text) + self.assertIn('(( 10#$PREFILL_FIRST_BURST_STEPS > 1024 ))', text) + launch_start = text.index("launch_command=(env", text.index("else\n launch_command=")) + burst_assignment = text.index( + 'DFLASH_PREFILL_FIRST_BURST_STEPS="$PREFILL_FIRST_BURST_STEPS"', + launch_start, + ) + launch_end = text.index('"${command[@]}")', burst_assignment) + self.assertLess(launch_start, burst_assignment) + self.assertLess(burst_assignment, launch_end) + + def test_runners_record_bounded_idle_prefill_budget(self) -> None: + ragged = (HERE / "run_qwen36_concurrency.sh").read_text(encoding="utf-8") + canonical = ( + HERE / "run_qwen36_canonical_concurrency.sh" + ).read_text(encoding="utf-8") + for text in (ragged, canonical): + self.assertIn( + 'IDLE_PREFILL_TOKENS="${IDLE_PREFILL_TOKENS:-4096}"', text + ) + self.assertIn('^[0-9]{1,4}$ ]]', text) + self.assertIn('^[1-9][0-9]{0,4}$ ]]', text) + self.assertIn('(( 10#$IDLE_PREFILL_TOKENS > 16384 ))', text) + self.assertIn( + "IDLE_PREFILL_TOKENS must be an integer in range 1..16384", text + ) + self.assertIn('"idle_prefill_tokens"', text) + assignment = 'DFLASH_IDLE_PREFILL_TOKENS="$IDLE_PREFILL_TOKENS"' + self.assertEqual(ragged.count(assignment), 1) + self.assertEqual(canonical.count(assignment), 2) + self.assertIn( + '"idle_prefill_tokens":int(idle_prefill_tokens) ' + 'if variant != "llama" else None', + ragged, + ) + self.assertIn( + '"idle_prefill_tokens":int(idle_prefill_tokens)', canonical + ) + + def test_ragged_runner_rejects_oversized_tuning_integers(self) -> None: + runner = HERE / "run_qwen36_concurrency.sh" + oversized = "18446744073709551616" + cases = ( + ( + "PREFILL_FIRST_BURST_STEPS", + "PREFILL_FIRST_BURST_STEPS must be an integer in range 0..1024", + ), + ( + "IDLE_PREFILL_TOKENS", + "IDLE_PREFILL_TOKENS must be an integer in range 1..16384", + ), + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for variable, message in cases: + env = { + "PATH": os.environ.get("PATH", ""), + "MODEL": "/dev/null", + "LUCE_SERVER_BIN": "/bin/true", + "LLAMA_SERVER_BIN": "/bin/true", + "OUT": str(root / variable), + variable: oversized, + } + result = subprocess.run( + [str(runner)], env=env, text=True, capture_output=True, check=False + ) + with self.subTest(variable=variable): + self.assertEqual(result.returncode, 2) + self.assertIn(message, result.stderr) + + def test_ragged_runner_can_fail_closed_on_gpu_arch(self) -> None: + text = (HERE / "run_qwen36_concurrency.sh").read_text(encoding="utf-8") + self.assertIn('EXPECTED_GPU_ARCH="${EXPECTED_GPU_ARCH:-}"', text) + self.assertIn('rocminfo 2>/dev/null | grep -F -- "$EXPECTED_GPU_ARCH"', text) + self.assertIn('"$case_dir/gpu-identity.txt"', text) + self.assertIn('"$binary" --list-devices', text) + self.assertIn('"$case_dir/llama-list-devices.txt"', text) + self.assertIn('-ngl all -lv 4 --reasoning off --reasoning-format none', text) + self.assertIn('offloaded ([1-9][0-9]*)/([1-9][0-9]*) layers to GPU', text) + self.assertIn('"$case_dir/server-gpu-proof.txt"', text) + self.assertIn('"expected_gpu_arch"', text) + class SummarizerTests(unittest.TestCase): @staticmethod @@ -163,6 +306,49 @@ def test_summary_reports_ttft_median_and_max(self) -> None: self.assertEqual(row.split("|")[9].strip(), "1.000") self.assertEqual(row.split("|")[10].strip(), "2.000") + def test_summary_reports_optional_native_prefill_metrics(self) -> None: + luce = self.item("luce-k8", 20.0) + luce["level"].update({ + "server_native_prefilled_tokens_total": 4096, + "server_native_prefill_window_ms": 1234.5, + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": True, + "server_native_prefill_tokens_per_s": 321.25, + }) + text = summarizer.summarize([luce, self.item("llama", 8.0)]) + self.assertIn("Native prefill tok/s | Native prefill window ms", text) + luce_row = next( + line for line in text.splitlines() if "| luce-k8 |" in line + ) + llama_row = next( + line for line in text.splitlines() if "| llama |" in line + ) + self.assertEqual(luce_row.split("|")[-3].strip(), "321.25") + self.assertEqual(luce_row.split("|")[-2].strip(), "1234.5") + self.assertEqual(llama_row.split("|")[-3].strip(), "n/a") + self.assertEqual(llama_row.split("|")[-2].strip(), "n/a") + + def test_summary_rejects_mismatched_native_completeness_flags(self) -> None: + item = self.item("luce-k8", 20.0) + item["level"].update({ + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": False, + }) + with self.assertRaisesRegex(ValueError, "mismatched native prefill"): + summarizer.summarize([item]) + + def test_summary_rejects_partial_native_repeats(self) -> None: + complete = self.item("luce-k8", 20.0, repeat=1) + complete["level"].update({ + "server_native_prefill_window_ms": 100.0, + "server_native_prefill_tokens_per_s": 200.0, + "server_native_prefill_token_count_complete": True, + "server_native_prefill_timing_complete": True, + }) + missing = self.item("luce-k8", 21.0, repeat=2) + with self.assertRaisesRegex(ValueError, "partial native prefill telemetry"): + summarizer.summarize([complete, missing]) + def test_load_reports_rejects_missing_prompt_usage(self) -> None: report = { "ignore_eos": True, diff --git a/harness/benchmarks/concurrency/test_concurrent_benchmark.py b/harness/benchmarks/concurrency/test_concurrent_benchmark.py index e0b337dc4..3d0e94906 100644 --- a/harness/benchmarks/concurrency/test_concurrent_benchmark.py +++ b/harness/benchmarks/concurrency/test_concurrent_benchmark.py @@ -28,6 +28,14 @@ def test_sse_parser_handles_events_and_done(self) -> None: ['{"choices":[{"delta":{"content":"hi"}}]}', "[DONE]"], ) + def test_default_matrix_starts_at_c2(self) -> None: + self.assertEqual(benchmark.DEFAULT_CLIENT_LEVELS, (2, 4, 8, 16)) + + def test_run_rejects_duplicate_client_levels(self) -> None: + args = argparse.Namespace(client_levels=[2, 4, 2]) + with self.assertRaisesRegex(ValueError, "distinct"): + benchmark.run(args) + def test_prompt_selection_never_wraps(self) -> None: self.assertEqual(benchmark.request_prompts(["a", "b", "c"], 2, 1), ["b", "c"]) with self.assertRaisesRegex(ValueError, "refusing to reuse"): @@ -41,11 +49,14 @@ def test_level_uses_exact_usage_and_first_token_window(self) -> None: prompt_counts = iter((10, 30)) def fake_request(_args: argparse.Namespace, prompt: str) -> dict: + prompt_count = next(prompt_counts) start = time.perf_counter() return { "t_start": start, "t_first": start + 0.5, "t_end": start + 1.0, "duration_s": 1.0, "ttft_s": 0.5, "decode_duration_s": 0.5, - "completion_tokens": 8, "prompt_tokens": next(prompt_counts), + "completion_tokens": 8, "prompt_tokens": prompt_count, + "server_native_prefilled_tokens": prompt_count, + "server_native_prefill_ms": 100.0 if prompt_count == 10 else 200.0, "finish_reason": "length", "error": None, "content_sha256": benchmark.sha256_text(prompt + " output"), "reasoning_content_sha256": benchmark.sha256_text(""), @@ -68,6 +79,21 @@ def fake_request(_args: argparse.Namespace, prompt: str) -> dict: level["prompt_tokens_per_s_to_first_token"], 40 / level["prompt_to_first_token_s"], ) + self.assertEqual(level["server_native_prefilled_tokens_total"], 40) + self.assertEqual(level["server_native_prefill_ms_max"], 200.0) + expected_window_ms = max( + record["start_offset_s"] * 1000.0 + record["server_native_prefill_ms"] + for record in level["requests_detail"] + ) + self.assertAlmostEqual( + level["server_native_prefill_window_ms"], expected_window_ms + ) + self.assertAlmostEqual( + level["server_native_prefill_tokens_per_s"], + 40_000.0 / expected_window_ms, + ) + self.assertTrue(level["server_native_prefill_token_count_complete"]) + self.assertTrue(level["server_native_prefill_timing_complete"]) def test_stream_request_keeps_usage_separate_from_sse_chunks(self) -> None: class Response: @@ -77,7 +103,7 @@ def __iter__(self): return iter([ b'data: {"choices":[{"delta":{"content":"one chunk"}}]}\n', b"\n", b'data: {"choices":[{"delta":{},"finish_reason":"length"}]}\n', b"\n", - b'data: {"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":64}}\n', b"\n", + b'data: {"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":64,"timings":{"prefilled_tokens":9,"prefill_ms":30.0}}}\n', b"\n", b"data: [DONE]\n", b"\n", ]) @@ -93,6 +119,29 @@ def __iter__(self): self.assertIsNone(record["error"]) self.assertIsNotNone(record["request_decode_tok_s"]) self.assertEqual(record["content_sha256"], benchmark.sha256_text("one chunk")) + self.assertEqual(record["server_native_prefilled_tokens"], 9) + self.assertEqual(record["server_native_prefill_ms"], 30.0) + self.assertEqual(record["server_native_prefill_tokens_per_s"], 300.0) + + def test_missing_prompt_usage_invalidates_fixed_workload(self) -> None: + def fake_request(_args: argparse.Namespace, prompt: str) -> dict: + start = time.perf_counter() + return { + "t_start": start, "t_first": start + 0.5, "t_end": start + 1.0, + "duration_s": 1.0, "ttft_s": 0.5, "decode_duration_s": 0.5, + "completion_tokens": 8, "prompt_tokens": None, + "finish_reason": "length", "error": None, + "content_sha256": benchmark.sha256_text(prompt + " output"), + "reasoning_content_sha256": benchmark.sha256_text(""), + "content_chars": 6, "reasoning_content_chars": 0, + "request_output_tok_s": 8.0, "request_decode_tok_s": 14.0, + } + + args = argparse.Namespace(max_tokens=8, ignore_eos=True, timeout=2.0) + with mock.patch.object(benchmark, "stream_request", side_effect=fake_request): + level = benchmark.run_level(1, args, ["prompt"], 0) + self.assertFalse(level["prompt_token_count_complete"]) + self.assertFalse(level["fixed_token_workload_valid"]) def test_stream_request_preserves_canonical_message_roles(self) -> None: captured = {} @@ -119,8 +168,26 @@ def fake_open(request, timeout): {"role": "user", "content": "user"}, ] with mock.patch.object(benchmark.urllib.request, "urlopen", side_effect=fake_open): - benchmark.stream_request(args, messages) + record = benchmark.stream_request(args, messages) self.assertEqual(captured["payload"]["messages"], messages) + self.assertIsNone(record["server_native_prefilled_tokens"]) + self.assertIsNone(record["server_native_prefill_ms"]) + self.assertIsNone(record["server_native_prefill_tokens_per_s"]) + + def test_native_prefill_values_rejects_partial_or_invalid_shapes(self) -> None: + self.assertEqual(benchmark.native_prefill_values({}), (None, None)) + self.assertEqual( + benchmark.native_prefill_values({ + "timings": {"prefilled_tokens": True, "prefill_ms": "10"}, + }), + (None, None), + ) + self.assertEqual( + benchmark.native_prefill_values({ + "timings": {"prefilled_tokens": 4, "prefill_ms": None}, + }), + (4, None), + ) def test_stream_request_rejects_clean_eof_without_done(self) -> None: class Response: diff --git a/harness/benchmarks/concurrency/test_synthetic_runner_gpu_proof.py b/harness/benchmarks/concurrency/test_synthetic_runner_gpu_proof.py new file mode 100644 index 000000000..7fa3a8dc2 --- /dev/null +++ b/harness/benchmarks/concurrency/test_synthetic_runner_gpu_proof.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""End-to-end GPU-evidence checks for the paired synthetic runner.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +RUNNER = HERE / "run_qwen36_concurrency.sh" + + +FAKE_SERVER = """\ +#!/usr/bin/env python3 +import os +import signal +import sys +import time + +if "--version" in sys.argv[1:]: + print("version: 1 (4cb22cd)") + raise SystemExit(0) +if "--list-devices" in sys.argv[1:]: + print(os.environ.get("FAKE_LIST_DEVICES", "Available devices:\\n ROCm0: Radeon 8060S Graphics")) + raise SystemExit(0) + +line = os.environ.get("FAKE_SERVER_LOG", "") +if line: + print(line, flush=True) +signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) +signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) +while True: + time.sleep(60) +""" + + +class SyntheticRunnerGpuProofTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.bin_dir = self.root / "bin" + self.bin_dir.mkdir() + self.model = self.root / "model.gguf" + self.model.write_bytes(b"fake model") + self.noop = self.root / "noop.py" + self.noop.write_text("raise SystemExit(0)\n", encoding="utf-8") + self.luce = self._write_executable("luce-server", FAKE_SERVER) + self.llama = self._write_executable("llama-server", FAKE_SERVER) + self._write_executable("curl", "#!/bin/sh\nsleep 0.1\nexit 0\n") + self._write_executable( + "rocminfo", "#!/bin/sh\nprintf ' Name: gfx1151\\n'\n" + ) + + def tearDown(self) -> None: + self.temp.cleanup() + + def _write_executable(self, name: str, text: str) -> Path: + path = self.bin_dir / name + path.write_text(textwrap.dedent(text), encoding="utf-8") + path.chmod(0o755) + return path + + def _environment(self, out: Path, variant: str, server_log: str) -> dict[str, str]: + env = { + key: value + for key, value in os.environ.items() + if not re.match( + r"^(GGML_|DFLASH_|LUCE_|HIP_|ROCR_|HSA_|LD_PRELOAD$|LD_LIBRARY_PATH$)", + key, + ) + } + env.update( + PATH=f"{self.bin_dir}{os.pathsep}{env['PATH']}", + MODEL=str(self.model), + LUCE_SERVER_BIN=str(self.luce), + LLAMA_SERVER_BIN=str(self.llama), + OUT=str(out), + WORKLOADS="short", + VARIANTS=variant, + CLIENTS="2", + SLOTS="2", + REPEATS="1", + GPU_DEVICE="1", + EXPECTED_GPU_ARCH="gfx1151", + COOLDOWN_SECONDS="0", + HEALTH_TIMEOUT_SECONDS="2", + CLIENT=str(self.noop), + SUMMARIZER=str(self.noop), + FAKE_SERVER_LOG=server_log, + ) + return env + + def _run(self, name: str, variant: str, server_log: str, **extra: str): + out = self.root / name + env = self._environment(out, variant, server_log) + env.update(extra) + result = subprocess.run( + [str(RUNNER)], env=env, text=True, capture_output=True, check=False + ) + return result, out / "short" / "c2" / "r1" / variant + + def test_luce_and_llama_store_independent_runtime_evidence(self) -> None: + luce_result, luce_case = self._run( + "luce-ok", "luce-k8", "Device 0: Radeon 8060S Graphics, gfx1151" + ) + self.assertEqual(luce_result.returncode, 0, luce_result.stderr) + self.assertIn("gfx1151", (luce_case / "gpu-identity.txt").read_text()) + self.assertIn("gfx1151", (luce_case / "server-gpu-proof.txt").read_text()) + self.assertFalse((luce_case / "llama-list-devices.txt").exists()) + + llama_result, llama_case = self._run( + "llama-ok", "llama", "llama_model_load: offloaded 65/65 layers to GPU" + ) + self.assertEqual(llama_result.returncode, 0, llama_result.stderr) + self.assertIn("gfx1151", (llama_case / "gpu-identity.txt").read_text()) + self.assertIn("ROCm0:", (llama_case / "llama-list-devices.txt").read_text()) + self.assertIn( + str(self.llama), + (llama_case / "llama-list-devices-command.txt").read_text(), + ) + llama_command = (llama_case / "server-command.txt").read_text() + self.assertIn("-ngl all -lv 4", llama_command) + self.assertIn("--reasoning off --reasoning-format none", llama_command) + self.assertEqual( + (llama_case / "server-gpu-proof.txt").read_text().strip(), + "llama_model_load: offloaded 65/65 layers to GPU", + ) + + def test_llama_rejects_missing_rocm_device_and_partial_or_zero_offload(self) -> None: + no_device, _ = self._run( + "llama-no-device", + "llama", + "llama_model_load: offloaded 65/65 layers to GPU", + FAKE_LIST_DEVICES="Available devices:\n CPU: host", + ) + self.assertEqual(no_device.returncode, 1) + self.assertIn("did not expose a ROCm device", no_device.stderr) + + for name, line in ( + ("partial", "llama_model_load: offloaded 64/65 layers to GPU"), + ("zero", "llama_model_load: offloaded 0/0 layers to GPU"), + ): + with self.subTest(name=name): + result, case = self._run(f"llama-{name}", "llama", line) + self.assertEqual(result.returncode, 1) + self.assertIn("did not report a positive full GPU offload", result.stderr) + self.assertFalse((case / "server-gpu-proof.txt").exists()) + + def test_luce_rejects_preflight_runtime_identity_mismatch(self) -> None: + result, case = self._run( + "luce-wrong-process", "luce-k8", "Device 0: Radeon AI PRO, gfx1201" + ) + self.assertEqual(result.returncode, 1) + self.assertIn("did not report expected GPU architecture gfx1151", result.stderr) + self.assertIn("gfx1151", (case / "gpu-identity.txt").read_text()) + self.assertEqual((case / "server-gpu-proof.txt").read_text(), "") + + +if __name__ == "__main__": + unittest.main()