diff --git a/.jaiph/docs_parity.jh b/.jaiph/docs_parity.jh index 10209f1c..30bf62cc 100755 --- a/.jaiph/docs_parity.jh +++ b/.jaiph/docs_parity.jh @@ -1,29 +1,31 @@ #!/usr/bin/env jaiph const role = """ - You are an expert technical writer for this project. - 1. You are fluent in Markdown and can read TypeScript code and Bash - 2. You write for a developer audience, focusing on clarity and practical - examples. - 3. You are concise, specific, and value dense - 4. Write so that a new developer to this codebase can understand your - writing, but don't assume your audience are experts in the topic/area you - are writing about. - 5. You are good in formulating generic context and describing the problem - starting from the generic part, leaving the specific details for the - last step, once the audience is aware of the generic context and the - problem. - 6. You write problem explanation and goals in a human approachable way, - while keeping details dense in separate sections, so both human and AI - 7. Source code and docs/architecture.md are the single source of truth. You don't - trust the existing documentation blindly. + You are an expert technical writer for this project. + 1. You are fluent in Markdown and can read TypeScript code and Bash + 2. You write for a developer audience, focusing on clarity and practical + examples. + 3. You are concise, specific, and value dense + 4. Write so that a new developer to this codebase can understand your + writing, but don't assume your audience are experts in the topic/area you + are writing about. + 5. You are good in formulating generic context and describing the problem + starting from the generic part, leaving the specific details for the + last step, once the audience is aware of the generic context and the + problem. + 6. You write problem explanation and goals in a human approachable way, + while keeping details dense in separate sections, so both human and AI + 7. Source code and docs/architecture.md are the single source of truth. You don't + trust the existing documentation blindly. """ script assert_newline_paths_are_files = ``` -while IFS= read -r f; do - [ -z "$f" ] && continue - test -f "$f" || return 1 -done <<< "$1" + while IFS= read -r f; do + f="${f#"${f%%[![:space:]]*}"}" + f="${f%"${f##*[![:space:]]}"}" + [ -z "$f" ] && continue + test -f "$f" || return 1 + done <<< "$1" ``` rule docs_files_present(list) { @@ -31,20 +33,20 @@ rule docs_files_present(list) { } script assert_worktree_clean_for_docs = ``` -local current_changed_files -current_changed_files="$( - { - git diff --name-only --cached - git diff --name-only - git ls-files --others --exclude-standard - } | sort -u -)" -if [ -n "$current_changed_files" ]; then - echo "Refusing to run docs parity workflow on a dirty worktree." >&2 - echo "Please commit, stash, or discard these files first:" >&2 - echo "$current_changed_files" >&2 - return 1 -fi + local current_changed_files + current_changed_files="$( + { + git diff --name-only --cached + git diff --name-only + git ls-files --others --exclude-standard + } | sort -u + )" + if [ -n "$current_changed_files" ]; then + echo "Refusing to run docs parity workflow on a dirty worktree." >&2 + echo "Please commit, stash, or discard these files first:" >&2 + echo "$current_changed_files" >&2 + return 1 + fi ``` rule worktree_is_clean() { @@ -52,58 +54,50 @@ rule worktree_is_clean() { } script assert_only_allowed_changed = ``` -local allowed="$1" -local after_changed_files -after_changed_files="$( - { - git diff --name-only --cached - git diff --name-only - git ls-files --others --exclude-standard - } | sort -u -)" -while IFS= read -r changed_file; do - [ -z "$changed_file" ] && continue - if [[ $'\n'"$allowed"$'\n' == *$'\n'"$changed_file"$'\n'* ]]; then - continue - fi - echo "Unexpected file changed by docs prompt: $changed_file" >&2 - return 1 -done <<< "$after_changed_files" + local allowed="$1" + local after_changed_files + after_changed_files="$( + { + git diff --name-only --cached + git diff --name-only + git ls-files --others --exclude-standard + } | sort -u + )" + while IFS= read -r changed_file; do + [ -z "$changed_file" ] && continue + if [[ $'\n'"$allowed"$'\n' == *$'\n'"$changed_file"$'\n'* ]]; then + continue + fi + echo "Unexpected file changed by docs prompt: $changed_file" >&2 + return 1 + done <<< "$after_changed_files" ``` rule only_expected_docs_changed_after_prompt(allowed) { run assert_only_allowed_changed(allowed) } -script first_line_str = `printf '%s\n' "$1" | head -n 1` - -script rest_lines_str = `printf '%s\n' "$1" | tail -n +2` - script list_docs_md_paths = ``` -local out="" -local f -for f in docs/*.md; do - out="${out:+$out -}$f" -done -printf '%s\n' "$out" + local out="" f + for f in docs/*.md; do + if [ -z "$out" ]; then + out="$f" + else + out="$out"$'\n'"$f" + fi + done + printf '%s\n' "$out" ``` script build_allowed_paths_block = ``` -local out="README.md -docs/index.html -docs/_layouts/docs.html -src/cli/shared/usage.ts" -local f -for f in docs/*.md; do - out="$out -$f" -done -printf '%s\n' "$out" + local out f + out="$(printf '%s\n' README.md docs/index.html docs/_layouts/docs.html src/cli/shared/usage.ts)" + for f in docs/*.md; do + out="$out"$'\n'"$f" + done + printf '%s\n' "$out" ``` -script join_newline_args = `printf '%s\n' "$@"` - workflow update_from_task(taskDesc) { prompt """ @@ -124,7 +118,7 @@ workflow update_from_task(taskDesc) { The task description is: ${taskDesc} -""" + """ } workflow docs_page(path) { @@ -206,31 +200,16 @@ workflow docs_overview(docPaths) { """ } -workflow process_docs_md_recursive(file, remaining) { - run docs_page(file) - if remaining == "" { - return - } - const next = run first_line_str(remaining) - const rest = run rest_lines_str(remaining) - run process_docs_md_recursive(next, rest) -} - -workflow maybe_process_docs_md(first_doc, rest_docs) { - if first_doc == "" { - return - } - run process_docs_md_recursive(first_doc, rest_docs) -} - workflow default() { ensure worktree_is_clean() const allowed_list = run build_allowed_paths_block() ensure docs_files_present(allowed_list) const docs_md_list = run list_docs_md_paths() - const first_doc = run first_line_str(docs_md_list) - const rest_docs = run rest_lines_str(docs_md_list) - run maybe_process_docs_md(first_doc, rest_docs) + for path in docs_md_list { + if path != "" { + run docs_page(path) + } + } run docs_overview(docs_md_list) ensure only_expected_docs_changed_after_prompt(allowed_list) } diff --git a/CHANGELOG.md b/CHANGELOG.md index add83fb6..be0622c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,49 @@ # Unreleased -- **Cleanup — remove `JAIPH_TEST_MODE` event suppression from production runtime code:** `RuntimeEventEmitter.emitStep` / `emitLog` no longer read `this.env.JAIPH_TEST_MODE` to decide whether to write `__JAIPH_EVENT__` lines to stderr. A construction-time `suppressLiveEvents?: boolean` option replaces the per-call env check: `NodeWorkflowRuntime` accepts it in its options and forwards it to `RuntimeEventEmitter`. `node-test-runner.ts` passes `suppressLiveEvents: true` when constructing the in-process runtime for `test_run_workflow` steps so `node --test` reporter output stays clean. `JAIPH_TEST_MODE: "1"` is still set in the test runner's env — but only for `prompt.ts`'s mock-mode selection, not event emission. No other production caller constructs `NodeWorkflowRuntime` directly, so the spawned `node-workflow-runner.js` child defaults to `suppressLiveEvents: false` and live events stream to stderr exactly as before. Durable `appendRunSummaryLine` writes to `run_summary.jsonl` are unchanged in either mode. Existing in-process unit tests under `node-workflow-runtime.artifacts.test.ts` pass the new option through their `NodeWorkflowRuntime` constructions. +- **Language:** `for in { … }` in workflows and rules iterates newline-delimited lines of a string binding. Newlines normalize `\r\n` to `\n`; a single trailing empty segment from a final newline is omitted. Lines are not trimmed and empty interior lines are still iterated unless the body skips them (e.g. `if line != "" { … }`). Documented in `docs/language.md`. +- **Tests / QA:** Unit tests for string line splitting (`src/runtime/string-lines.test.ts`); E2E `e2e/tests/135_for_string_lines.sh`. + +# 0.9.4 + +## Summary + +Maintenance and simplification: +- **Breaking:** Inbox dispatch is sequential only (parallel config/env removed). Stricter grammar: multiline `config` blocks only; no one-line braced workflows; no semicolon-separated statements in workflow/rule bodies. +- **Runtime:** Single-line shell steps run in the Node runtime (`sh -c`); script capture only on success; async `run` + `recover` return propagation fixed; mock prompts use JSON arm dispatch and an in-memory response queue; inbox artifact files are written only when a route consumes the channel. +- **CLI / install:** Failure footers use the **last** failed step in `run_summary.jsonl`; curl install ships `package.json` so stable installs resolve the correct default Docker image tag. +- **Language:** RHS bare identifiers and bare dotted identifiers are treated as interpolation sugar where applicable. +- **Library:** `artifacts.save(paths)` in single-argument form (path or newline-separated list); `git format-patch` workflows use `--stdout` so patch bytes are captured. +- **Repo:** `node-workflow-runtime` split into arg-parser, event-emitter, and mock modules; test directories consolidated under `integration/`, `test-fixtures/`, `test-infra/`; `JAIPH_TEST_MODE` no longer suppresses stderr events in runtime code (constructor option instead). +- **Docs / DX:** Agent-proxy design note; explicit parse error for `test` blocks outside `*.test.jh`; architecture/inbox corrections; getting-started shortened. +## All changes + +- **Breaking — Language:** Inline one-line `config { k = v }` is removed — only the multiline `config {\n … \n}` form parses (matches documented grammar). The formatter no longer emits compact inline `config`, which would be invalid input. Examples such as `examples/async.jh` were migrated. +- **Breaking — Language:** Single-line `workflow name() { stmt }` braced form removed; workflow and rule bodies require one statement per line as in the grammar. +- **Breaking — Language:** Semicolons no longer separate statements in workflow/rule bodies (`splitStatementsOnSemicolons` remains for `match` arms). Multiple statements on one line joined by `;` must be split across lines. +- **Breaking — Inbox dispatch is always sequential** — The optional parallel inbox mode is removed: there is no `run.inbox_parallel` config key, no `JAIPH_INBOX_PARALLEL` environment variable (it is ignored), and no `JAIPH_INBOX_PARALLEL_LOCKED` shim. Route targets for a queued message always run **one after another** in declaration order on the `channel` line, inside `NodeWorkflowRuntime`’s `drainWorkflowQueue`. Using `run.inbox_parallel = …` in a `config { … }` block is `E_PARSE: unknown config key: run.inbox_parallel`. Docs and E2E now match sequential-only semantics; unit tests cover the unknown key and parity of dispatch event order with and without the old env var set. E2E harness clears inherited `JAIPH_*` noise so CI stays reliable in polluted agent environments. +- **Language / Runtime:** Single-line shell steps execute via `sh -c` with script working-directory semantics in the Node workflow runtime (replacing the removed bash-era path for these steps). `validateReferences` and related checks were extended for `send` arrow targets, managed `run` on bare names, and dotted references. +- **Fix — Interpolation:** RHS values treat bare identifiers and bare dotted identifiers as `${…}` interpolation sugar where the grammar allows, so dotted env-style names behave consistently with other binding references. +- **Fix — Runtime return capture:** `executeScript` / `executeShLine` / `executeMockShellBody` return captured stdout only when the subprocess exits with status 0 (failed commands no longer leak stdout as a workflow return value). +- **Fix — Async recover:** `run … recover(e) { … }` now propagates `recoverReturn` through the implicit async join site (parity with synchronous `ensure` / catch semantics). +- **Language:** Reject `return 0`, `return $?`, and bare integer `return N` in workflows/rules with a clear diagnostic instead of emitting a useless shell line. +- **Runtime — Mock prompts:** Mock arms are passed structurally as JSON via `JAIPH_MOCK_PROMPT_ARMS_JSON` with in-process dispatch in `mock.ts` (no bash dispatcher). Sequential mock responses use `JAIPH_MOCK_RESPONSES_JSON` and an in-memory queue (`consumeNextMockResponse`), removing per-step file churn. +- **Runtime — Inbox files:** Inbox files under the run directory are written only when a route consumes the channel (no “audit-only” files for unrouted sends). +- **Fix — Mock shell:** `executeMockShellBody` uses `bash -c` instead of a tempfile indirection; removes an ESM/`require` shadowing bug in the mock shell path. +- **Library:** `jaiphlang/artifacts` exposes a single `save(paths)` workflow: one filesystem path or a newline-separated list; destination relpaths are derived per source (leading `./` stripped; absolute sources use `basename` only). The bundled engineer workflow uses `git.commit` plus `git format-patch` with `--stdout` / `HEAD` so the patch **contents** are saved (without `--stdout`, `format-patch` only printed the filename on stdout). +- **Parser / UX:** A `test { … }` block in a file whose name does not end in `.test.jh` now fails with `E_PARSE` explaining that test blocks belong in `*.test.jh` (instead of falling through to a generic unsupported-statement error). +- **Repo — Compiler/runtime cleanup:** Removed a large amount of dead bash-era kernel code and legacy parse rejects; consolidated import parsers and config-key handling; stricter top-level dispatch in `parser.ts`. `.jaiph/git.jh` moves to `jaiphlang/git` with `import "jaiphlang/git" as git`. Collapsed duplicate parser/runtime paths from the audit series (`B1`, `B10`, `B11`, etc.). +- **Repo — AST clarity (no source keyword changes):** AST field names now align with keywords: the single-shot branch is `step.catch`, the repair-and-retry loop body is `step.recover`. TypeScript uses `catchDef` where `catch` is reserved. Workflow source still uses `run foo() recover(e) { … }` and `run foo() catch(e) { … }`. +- **Fix — Runtime config seed:** Restore `cpSync` seeding of Claude config into the workspace fallback when only session env is unwritable (auth preservation). +- **Docs:** Add `design/2026-05-12-agent-proxy.md` (Phantom Token / credential proxy design for sandboxed agents). Update `architecture.md` (drop stale `run-step-exec` / `seq-alloc` references). Update `inbox.md` (remove unused dispatch env vars; document inbox files only when consumed). Shorten `getting-started` overview. +- **Tests / QA:** E2E and txtar fixtures for `import script` (shell/Python, capture, missing file); extended parse/validate error fixtures; QA scripts (`read_txtar_*`) point at `test-fixtures/compiler-txtar/`. +- **Repo:** `AUDIT_PROGRESS.md` removed (remaining items tracked in `QUEUE.md`). `Gemfile.lock` records `ffi` platform gems for arm64-darwin and x86_64-linux where needed. +- **Cleanup — remove `JAIPH_TEST_MODE` event suppression from production runtime code:** `RuntimeEventEmitter.emitStep` / `emitLog` no longer read `this.env.JAIPH_TEST_MODE` to decide whether to write `__JAIPH_EVENT__` lines to stderr. A construction-time `suppressLiveEvents?: boolean` option replaces the per-call env check: `NodeWorkflowRuntime` accepts it in its options and forwards it to `RuntimeEventEmitter`. `node-test-runner.ts` passes `suppressLiveEvents: true` when constructing the in-process runtime for `test_run_workflow` steps so `node --test` reporter output stays clean. `JAIPH_TEST_MODE: "1"` is still set in the test runner's env — but only for `prompt.ts`'s mock-mode selection, not event emission. No other production caller constructs `NodeWorkflowRuntime` directly, so the spawned `node-workflow-runner.js` child defaults to `suppressLiveEvents: false` and live events stream to stderr exactly as before. Durable `appendRunSummaryLine` writes to `run_summary.jsonl` are unchanged in either mode. Existing in-process unit tests under `node-workflow-runtime.artifacts.test.ts` pass the new option through their `NodeWorkflowRuntime` constructions. - **Repo — `node-workflow-runtime.ts` split:** The 1915-LoC `src/runtime/kernel/node-workflow-runtime.ts` god file is split into the orchestrator plus three focused sibling modules under `src/runtime/kernel/`. No behavior changes — pure relocation; existing tests pass unchanged (helpers re-imported from their new location where needed). - **`runtime-arg-parser.ts`** — every stateless free helper that used to live above the `NodeWorkflowRuntime` class (`interpolate`, `parseInlineCaptureCall`, `commaArgsToInterpolated`, `parseArgsRaw`, `parseInlineScriptAt`, `parseManagedArgAt`, `parseArgTokens`, `stripOuterQuotes`, `parsePromptSchema`, `sanitizeName`, `nowIso`), the `BARE_IDENT_RE` / `MAX_EMBED` / `MAX_RECURSION_DEPTH` constants, and the `ParsedArgToken` / `PromptSchemaField` types. Direct unit tests added in `runtime-arg-parser.test.ts`. - **`runtime-event-emitter.ts`** — `RuntimeEventEmitter` owns `emitWorkflow`, `emitStep`, `emitPromptStepStart`, `emitPromptStepEnd`, `emitPromptEvent`, `emitLog`, plus the monotonic step and prompt sequence counters. Constructed with `{ runId, runDir, env, getFrameStack, getAsyncIndices }`. No more direct `process.stderr.write(__JAIPH_EVENT__ …)` scattered through the runtime. - **`runtime-mock.ts`** — `executeMockBodyDef` and `executeMockShellBody` move here as exported functions taking `{ ref, args, env, cwd, executeStepsBack }` (the last is a callback so steps-kind mocks dispatch back into the runtime). The `require("node:child_process")` call that shadowed ESM imports inside `executeMockShellBody` is gone — replaced by a top-of-file `import`. - The orchestrator (`node-workflow-runtime.ts`) keeps the `NodeWorkflowRuntime` class, workflow/step orchestration (`runDefault`, `runNamedWorkflow`, `executeSteps`, `executeStep`, `runRecoverBody`, `runPromptStep`, frame and scope management), async-handle bookkeeping (`getAsyncIndices`, `getFrameStack`), and heartbeat (`startHeartbeat`, `stopHeartbeat`, `writeHeartbeat`). Dependency direction is one-way (orchestrator → helpers/emitter/mock); no circular imports. - -- **Breaking — Inbox dispatch is always sequential** — The optional parallel inbox mode is removed: there is no `run.inbox_parallel` config key, no `JAIPH_INBOX_PARALLEL` environment variable (it is ignored), and no `JAIPH_INBOX_PARALLEL_LOCKED` shim. Route targets for a queued message always run **one after another** in declaration order on the `channel` line, inside `NodeWorkflowRuntime`’s `drainWorkflowQueue`. Using `run.inbox_parallel = …` in a `config { … }` block is `E_PARSE: unknown config key: run.inbox_parallel`. Docs and E2E now match sequential-only semantics; unit tests cover the unknown key and parity of dispatch event order with and without the old env var set. - - **Fix — CLI failure footer:** `Output of failed step` and the footer `out:` / `err:` paths now resolve from the **last** non-zero `STEP_END` in `run_summary.jsonl` (append order), not the first. The first failure line could be a recovered `catch`/`ensure` attempt, a stray record, or unrelated noise; the last failure matches the terminal step (the one the progress tree marks as failed). **`src/cli/shared/errors.test.ts`** covers multiple non-zero `STEP_END` lines. - **Fix — Docker default image tag:** `curl` / `docs/install` copied only `dist/src` into `~/.local/bin/.jaiph`, so the CLI could not read `package.json` and defaulted the sandbox image to `ghcr.io/jaiphlang/jaiph-runtime:nightly` even for stable installs. The installer now copies `package.json` beside `src/`, and `resolveDefaultDockerImageTag` checks both the installer layout and the npm `dist/src/runtime` layout. - **Repo — Test directory consolidation:** Consolidated the five-way test directory split (`src/**/*.test.ts`, `test/`, `tests/`, `compiler-tests/`, `golden-ast/`) into three test "places" plus two clearly named support directories. File moves: diff --git a/README.md b/README.md index a5bed42c..baeb4b2c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ![Jaiph](docs/logo.png) -[jaiph.org](https://jaiph.org) · [Getting Started](docs/getting-started.md) ([jaiph.org/getting-started](https://jaiph.org/getting-started)) · [Setup](docs/setup.md) · [Libraries](docs/libraries.md) · [Grammar](docs/grammar.md) · [CLI](docs/cli.md) · [Configuration](docs/configuration.md) · [Testing](docs/testing.md) · [Hooks](docs/hooks.md) · [Inbox & Dispatch](docs/inbox.md) · [Sandboxing](docs/sandboxing.md) · [Runtime artifacts](docs/artifacts.md) · [Async Handles](docs/spec-async-handles.md) · [Architecture](docs/architecture.md) · [Contributing](docs/contributing.md) +[jaiph.org](https://jaiph.org) · [Getting Started](docs/getting-started.md) ([jaiph.org/getting-started](https://jaiph.org/getting-started)) · [Setup](docs/setup.md) · [Libraries](docs/libraries.md) · [Language](docs/language.md) · [Grammar](docs/grammar.md) · [CLI](docs/cli.md) · [Configuration](docs/configuration.md) · [Testing](docs/testing.md) · [Hooks](docs/hooks.md) · [Inbox & Dispatch](docs/inbox.md) · [Sandboxing](docs/sandboxing.md) · [Runtime artifacts](docs/artifacts.md) · [Async Handles](docs/spec-async-handles.md) · [Architecture](docs/architecture.md) · [Contributing](docs/contributing.md) --- @@ -16,6 +16,15 @@ > [!WARNING] > Jaiph is still in an early stage. Expect breaking changes. +## Features + +- **Workflows** — Compose `prompt`, `run`, `ensure`, channel sends, conditionals, `run async` with implicit join, `catch`, and repair-and-retry `recover`. +- **Rules and scripts** — Rules stay structured (no raw shell lines); **`script`** steps run bash or polyglot code as subprocesses. +- **Agents** — Backends include Cursor, Claude, Codex (HTTP), or a custom `agent.command`. +- **Testing** — `*.test.jh` files run in-process (`jaiph test`) with mocks and `expect_*` assertions ([Testing](docs/testing.md)). +- **Safety and inspectability** — Docker-backed sandbox for **`jaiph run`** (env-controlled; see [Sandboxing](docs/sandboxing.md)); live **`__JAIPH_EVENT__`** on stderr and durable **`.jaiph/runs/`** artifacts ([Architecture](docs/architecture.md)). +- **Tooling** — `jaiph compile`, `jaiph format`, `jaiph install` / `.jaiph/libs/`, and optional `hooks.json` ([CLI](docs/cli.md), [Hooks](docs/hooks.md)). + ## Core components - **CLI** (`src/cli`) — `jaiph run` / `test` / `compile` / `format` / `init` / `install` / `use`; prepares scripts, spawns the workflow runner (or in-process test runner), parses `__JAIPH_EVENT__` on stderr, runs hooks on `jaiph run` only. @@ -53,7 +62,18 @@ Or install from npm: npm install -g jaiph ``` -Verify: `jaiph --version`. Switch versions: `jaiph use nightly` or `jaiph use 0.9.3`. +Verify: `jaiph --version`. Switch versions: `jaiph use nightly` or `jaiph use 0.9.4`. + +Initialize a project (optional): `jaiph init` writes `.jaiph/` with bootstrap workflow, gitignore entries for runs/tmp, and **`SKILL.md`** when the CLI resolves a skill file on disk (`JAIPH_SKILL_PATH`, install-relative `jaiph-skill.md`, or `docs/jaiph-skill.md` under cwd — see [Setup](docs/setup.md)). Canonical skill text for agents: `https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md`. + +## Usage + +- Run the default workflow: `jaiph run path/to/main.jh [args...]` or `./main.jh [args...]` with a `#!/usr/bin/env jaiph` shebang. +- Run tests: `jaiph test` (workspace), `jaiph test ./dir`, or `jaiph test path.test.jh`. +- Validate without executing: `jaiph compile …` (same `validateReferences` checks as before `jaiph run`; no `scripts/` emission — see [Architecture](docs/architecture.md)). +- Format sources: `jaiph format …` / `jaiph format --check …`. + +Full flags and environment variables: [CLI reference](docs/cli.md). Doc map: [Getting Started](docs/getting-started.md). ## Example diff --git a/docs/Gemfile b/docs/Gemfile index d5d2130d..983cec24 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -1,4 +1,7 @@ source "https://rubygems.org" +# Ruby 4.0+ bundles some stdlib as gems; Jekyll 3 / Liquid / safe_yaml still require them. +gem "base64" +gem "bigdecimal" gem "jekyll", "~> 3.9" gem "kramdown-parser-gfm" gem "jekyll-relative-links" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 0301b67c..ad1195ae 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -3,6 +3,8 @@ GEM specs: addressable (2.8.9) public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + bigdecimal (4.1.2) colorator (1.1.0) concurrent-ruby (1.3.6) csv (3.3.5) @@ -72,6 +74,8 @@ PLATFORMS x86_64-linux DEPENDENCIES + base64 + bigdecimal jekyll (~> 3.9) jekyll-redirect-from jekyll-relative-links diff --git a/docs/_layouts/docs.html b/docs/_layouts/docs.html index e0d98280..bb4f5fd2 100644 --- a/docs/_layouts/docs.html +++ b/docs/_layouts/docs.html @@ -52,6 +52,7 @@
  • CLI
  • Configuration
  • Testing
    • +
  • Async handles
  • Inbox
  • Hooks
  • Sandboxing
    • diff --git a/docs/architecture.md b/docs/architecture.md index 8b8a9e2d..55e9ff50 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,9 @@ redirect_from: Jaiph is a workflow system with a **TypeScript CLI** and a **JavaScript kernel** (`src/runtime/kernel/`) that interprets the workflow AST in process — there is no separate “workflow shell” emitted for execution. -This page describes **how Jaiph is built**: repository layout of major subsystems, **core components**, compile and run pipelines, and **runtime contracts** (events, artifacts on disk, distribution). It is the map of the implementation. +This page describes **how Jaiph is built**: repository layout of major subsystems, **core components**, compile and run pipelines, and **runtime contracts** (events, artifacts on disk, distribution). It is the map of the implementation. For workflow syntax and semantics, see the [Language](language.md) guide; this document stays on implementation boundaries. + +**Why this split:** the transpiler turns each `script` block (and inline script bodies) into real files under `scripts/` with a stable layout and `JAIPH_SCRIPTS`, while **`NodeWorkflowRuntime` always executes from the AST** (`buildRuntimeGraph`). That separation keeps bash entrypoints predictable for subprocesses without duplicating workflow logic in a second language. For **how to contribute** — branches, test layers, E2E assertion policy, and bash harness details — see [Contributing](contributing.md). For the `*.test.jh` **language** and test blocks, see [Testing](testing.md). @@ -18,18 +20,20 @@ For **how to contribute** — branches, test layers, E2E assertion policy, and b Workflow authors write `.jh` / `.test.jh` modules. The toolchain turns those files into **validated** modules plus **extracted script files**, then the **same AST interpreter** runs workflows whether you use local `jaiph run`, Docker, or `jaiph test`. 1. Parse source into AST (the CLI parses once up front for `jaiph run` metadata such as `runtime` config; `buildRuntimeGraph` and transpilation use the same parser on disk contents). -2. **Compile-time** validation (`validateReferences`, invoked from **`emitScriptsForModule`** / **`buildScripts()`**) runs before script extraction, not inside `buildRuntimeGraph()` (the graph loader only parses modules and follows imports). The **`jaiph compile`** command runs the same validation over files or directories without executing workflows (see `src/cli/commands/compile.ts`). +2. **Compile-time** validation (`validateReferences`, invoked from **`emitScriptsForModule`** / **`buildScripts()`**) runs before script extraction, not inside `buildRuntimeGraph()` (the graph loader only parses modules and follows imports). The **`jaiph compile`** command walks the same import closure but runs **`validateReferences` only**: it parses each reachable module on disk and **does not** emit **`scripts/`** (no **`buildScriptFiles`** / **`buildScripts`**), **does not** invoke **`buildRuntimeGraph()`**, and never spawns the workflow runner (`src/cli/commands/compile.ts`). For a **directory** argument it discovers `*.jh` via `walkjhFiles`, which **skips** `*.test.jh`; to validate a test module, pass that file explicitly. Imported modules in the closure are still validated recursively either way. 3. **CLI** (`dist/src/cli.js` via npm, or a **Bun-compiled** `dist/jaiph` binary) prepares script executables (scripts-only), then spawns a **detached child** that loads **`node-workflow-runner.js`**. That child calls `buildRuntimeGraph()` and runs **`NodeWorkflowRuntime`**. The child’s interpreter is **`process.execPath`** of the CLI process (Node when you run `node dist/src/cli.js`, the standalone Bun binary when you run `dist/jaiph`). Script steps execute as managed subprocesses; prompt, inbox I/O, and event/summary emission are handled by the kernel under `src/runtime/kernel/`. 4. Stream live events to the CLI and persist durable run artifacts. +Interactive **`jaiph run`** parses **`__JAIPH_EVENT__`** lines from the runner’s stderr, renders the progress tree, and runs hooks. **`jaiph run --raw`** skips that shell: the child uses inherited stdio so events still land on stderr unchanged — used when embedding Jaiph or when the host wraps a container (see [CLI — `jaiph run`](cli.md#jaiph-run) and [Sandboxing — Docker container isolation](sandboxing.md#docker-container-isolation)). + All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`** — uses the **Node workflow runtime** (AST interpreter). Docker containers run the same `node-workflow-runner` process with the compiled JS source tree and scripts mounted read-only. ## Core components -- **CLI (`src/cli`)** - - Entry point (`run`, `test`, `compile`, `init`, `install`, `use`, `format`). - - **Workflow launch** is owned in TypeScript (`src/runtime/kernel/workflow-launch.ts` + `src/cli/run/lifecycle.ts`): spawns **`node-workflow-runner.js`** with `process.execPath`, which calls `buildRuntimeGraph()` then `NodeWorkflowRuntime`. `setupRunSignalHandlers` accepts an optional `onSignalCleanup` callback for Docker sandbox teardown on SIGINT/SIGTERM. - - Parses runtime events and renders progress; dispatches hooks. +- **CLI (`src/cli`, invoked via compiled `src/cli.ts` → `dist/src/cli.js`)** + - Entry point (`run`, `test`, `compile`, `init`, `install`, `use`, `format`). Paths ending in `.jh` / `.test.jh` are also accepted as implicit commands (see `src/cli/index.ts`). + - **Workflow launch** is owned in TypeScript (`src/runtime/kernel/workflow-launch.ts` + `src/cli/run/lifecycle.ts`): spawns **`node-workflow-runner.js`** with `process.execPath`, which calls `buildRuntimeGraph()` then `NodeWorkflowRuntime`. The **`jaiph run`** path always launches the **`default`** workflow via argv wired in `workflow-launch.ts` (`node-workflow-runner` calls `runDefault`). `setupRunSignalHandlers` accepts an optional `onSignalCleanup` callback for Docker sandbox teardown on SIGINT/SIGTERM. + - Parses runtime events and renders progress (except `--raw`); dispatches hooks. - **Parser (`src/parser.ts`, `src/parse/*`)** - Converts `.jh`/`.test.jh` into `jaiphModule` AST. @@ -39,24 +43,24 @@ All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`* - Shared compile-time schema (`jaiphModule`, step defs, test defs, hook payload types). - **Validator (`src/transpile/validate.ts`)** - - Resolves imports and symbol references; emits deterministic compile-time errors. Import resolution (`resolveImportPath` in `resolve.ts`) checks relative paths first, then falls back to project-scoped libraries under `/.jaiph/libs/` — the workspace root is threaded through all compilation call sites. Export visibility is enforced by `validateRef` in `validate-ref-resolution.ts`: if an imported module declares any `export`, only exported names are reachable through the import alias. + - Resolves imports and symbol references; emits deterministic compile-time errors. Import resolution (`resolveImportPath` in `transpile/resolve.ts`) checks relative paths first, then falls back to project-scoped libraries under `/.jaiph/libs/` — the workspace root is threaded through all compilation call sites. Export visibility is enforced by `validateRef` in `validate-ref-resolution.ts`: if an imported module declares any `export`, only exported names are reachable through the import alias. - **Transpiler (`src/transpiler.ts`, `src/transpile/*`)** - - **`emitScriptsForModule`** parses, runs **`validateReferences`**, and **`buildScriptFiles`** — the only compile path for `jaiph run` / `jaiph test` — **persists only atomic `script` files** under `scripts/`. Inline scripts (`` run `body`(args) ``) are also emitted as `scripts/__inline_` with deterministic hash-based names. There is no workflow-level bash emission. + - **`emitScriptsForModule`** parses, runs **`validateReferences`**, and **`buildScriptFiles`** — the only compile path for `jaiph run` / `jaiph test` — **persists only atomic `script` files** under `scripts/`. **`buildScripts()`** can also take a **directory** of non-test `*.jh` modules (`src/transpile/build.ts` uses `walkjhFiles`); the **`jaiph run`** and **`jaiph test`** commands always pass a **single entry file** (`.jh` or `*.test.jh`). Inline scripts (`` run `body`(args) ``) are also emitted as `scripts/__inline_` with deterministic hash-based names (`inlineScriptName` in `src/inline-script-name.ts`). There is no workflow-level bash emission. - **Node Workflow Runtime (`src/runtime/kernel/node-workflow-runtime.ts`)** - `NodeWorkflowRuntime` interprets the AST directly: walks workflow steps, manages scope/variables, delegates prompt and script execution to kernel helpers, handles channels/inbox/dispatch, owns the frame stack and heartbeat, and writes run artifacts. - Three sibling modules under `src/runtime/kernel/` carry concerns that used to live inline in the runtime file. Dependency direction is one-way (orchestrator → helpers/emitter/mock); no circular imports back. - **`runtime-arg-parser.ts`** — stateless interpolation and call-argument parsing (`interpolate`, `parseInlineCaptureCall`, `commaArgsToInterpolated`, `parseArgsRaw`, `parseInlineScriptAt`, `parseManagedArgAt`, `parseArgTokens`, `stripOuterQuotes`, `parsePromptSchema`, `sanitizeName`, `nowIso`) plus shared constants and the `ParsedArgToken` / `PromptSchemaField` types. Direct unit tests live in `runtime-arg-parser.test.ts`. - - **`runtime-event-emitter.ts`** — `RuntimeEventEmitter` owns the `__JAIPH_EVENT__` stderr stream and `run_summary.jsonl` writes for workflow/step/prompt/log events, plus the monotonic step and prompt sequence counters. Constructed with `{ runId, runDir, env, getFrameStack, getAsyncIndices, suppressLiveEvents? }`; the runtime delegates all event emission to it. The optional `suppressLiveEvents` flag (forwarded from `NodeWorkflowRuntime`'s `suppressLiveEvents` option) skips the live stderr write while leaving the durable `run_summary.jsonl` append intact — used by in-process callers like the test runner that share stderr with `node --test` reporter output. The CLI's spawned `node-workflow-runner` child does not set it, so production runs stream events to stderr as before. + - **`runtime-event-emitter.ts`** — `RuntimeEventEmitter` owns **`__JAIPH_EVENT__`** writes on stderr (step/log traffic when not suppressed), **`run_summary.jsonl`** appends for the wider timeline (including workflow/prompt records that are summary-first), plus step/prompt sequence counters. Constructed with `{ runId, runDir, env, getFrameStack, getAsyncIndices, suppressLiveEvents? }`; the runtime delegates structured emission to it. The optional `suppressLiveEvents` flag (forwarded from `NodeWorkflowRuntime`'s `suppressLiveEvents` option) skips the live stderr **`__JAIPH_EVENT__`** lines while **`appendRunSummaryLine`** keeps updating **`run_summary.jsonl`** — used by in-process callers like the test runner that share stderr with `node --test` reporter output. The CLI's spawned `node-workflow-runner` child does not set it, so production runs stream events to stderr as before. - **`runtime-mock.ts`** — `executeMockBodyDef` and `executeMockShellBody` for `*.test.jh` workflow/rule/script mocks. Shell-kind mocks run `bash -c`; steps-kind mocks dispatch back into the runtime via an `executeStepsBack` callback so the body runs against the full step interpreter. - - `buildRuntimeGraph()` (`graph.ts`) loads reachable modules with **`parsejaiph` only** (import closure); it does **not** run `validateReferences`. Cross-module refs are resolved from that graph at runtime. + - `buildRuntimeGraph()` (`graph.ts`) loads reachable modules with **`parsejaiph` only** (import closure); it does **not** run `validateReferences`. Cross-module refs are resolved from that graph at runtime. For **`script import`** declarations, `buildRuntimeGraph()` injects synthetic `ScriptDef` stubs (`graph.ts`) so reference resolution matches the validated compile path without re-reading external script bodies at graph-build time. - **Node Test Runner (`src/runtime/kernel/node-test-runner.ts`)** - Executes `*.test.jh` test blocks using `NodeWorkflowRuntime` with mock support (mock prompts, mock workflow/rule/script bodies). Pure Node harness — no Bash test transpilation. - **JS kernel (`src/runtime/kernel/`)** - - Prompt execution (`prompt.ts`), streaming parse (`stream-parser.ts`), schema (`schema.ts`), mocks (`mock.ts`), **`emit.ts`** (live `__JAIPH_EVENT__` + `run_summary.jsonl`), **`workflow-launch.ts`** (spawn contract). Script subprocesses are launched directly from `NodeWorkflowRuntime`. + - Prompt execution (`prompt.ts`), streaming parse (`stream-parser.ts`), schema (`schema.ts`), **`mock.ts`** (sequential prompt responses / mock-arm dispatch from test env JSON), **`runtime-mock.ts`** (mock workflow/rule/script **bodies** for `*.test.jh`), **`emit.ts`** (durable **`run_summary.jsonl`** helpers — `appendRunSummaryLine`, `formatUtcTimestamp` — consumed by `RuntimeEventEmitter`), **`workflow-launch.ts`** (spawn contract). **`RuntimeEventEmitter`** (`runtime-event-emitter.ts`) owns live **`__JAIPH_EVENT__`** lines on stderr and coordinates summary writes plus step/prompt sequence counters. Script subprocesses are launched directly from `NodeWorkflowRuntime`. - **Formatter (`src/format/emit.ts`)** - `jaiph format` rewrites `.jh` / `.test.jh` files into canonical style. Pure AST→text emitter; no side-effects beyond file writes. @@ -73,17 +77,17 @@ All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`* - Manage channels (`send`, routes, queue drain) through kernel logic. - Emit step/log events; persist run logs and summary timeline. - Prompt steps and managed script subprocesses: Node kernel owns execution, events, and control flow. -- Execute test blocks with mock support (`NodeTestRunner`). +- Execute test blocks with mock support (`runTestFile()` in `node-test-runner.ts`). ### CLI responsibilities - Parse, validate, and launch workflows/tests. - Own **process spawn** for `jaiph run` (detached workflow runner process group for signal propagation). -- Parse live runtime events; render terminal progress; trigger hooks. +- Parse live runtime events; render terminal progress; trigger hooks — skipped in **`jaiph run --raw`** (child stdio inherited; see [CLI](cli.md#jaiph-run)). ## Contracts -- **Live contract (runtime -> CLI):** `__JAIPH_EVENT__` JSON lines on **stderr only** — the single event channel for all modes (local and Docker). The CLI listens on stderr exclusively; stdout carries only plain script output. +- **Live contract (runtime → observing process):** `__JAIPH_EVENT__` JSON lines on **stderr only** — the structured event channel. Hooks and the interactive CLI consume that stream; see [Hooks](hooks.md). - **Durable contract:** `.jaiph/runs/...` + `run_summary.jsonl` (layout below). Channel transport remains file/queue based in runtime inbox logic. @@ -99,30 +103,31 @@ The runtime persists step captures and the event timeline under a UTC-dated hier / # UTC date (see NodeWorkflowRuntime) -/ # UTC time + JAIPH_SOURCE_FILE or entry basename 000001-module__step.out # stdout capture per step (6-digit seq prefix) - 000001-module__step.err # stderr capture (when non-empty) + 000001-module__step.err # stderr capture (may be empty) artifacts/ # user-published files (JAIPH_ARTIFACTS_DIR); created at run start - inbox/ # inbox message files (when channels are used) + inbox/ # audit copies of routed channel payloads (optional) heartbeat # liveness: epoch ms, refreshed about every 10s return_value.txt # when `jaiph run` default workflow returns a value (success only) run_summary.jsonl # durable event timeline ``` -Step sequence numbers are monotonic and unique per run: `NodeWorkflowRuntime` allocates them in memory when opening each step’s capture files (`%06d-.out|.err`). There is no `.seq` file in the run directory. +Step sequence numbers are monotonic and unique per run: `RuntimeEventEmitter` allocates them in memory (`allocStepSeq`) when opening each step’s capture files (`%06d-.out|.err`). There is no `.seq` file in the run directory. ## Channels and hooks in context -Channels are validated at compile time (`validateReferences` / send RHS rules) and executed via in-memory queue and dispatch in the Node runtime; durable inbox files under the run directory are for audit and reporting. See [Inbox & Dispatch](inbox.md). Hooks are CLI-only: they load from `hooks.json` and run as shell commands with JSON on stdin, driven by the same `__JAIPH_EVENT__` stream as the progress UI — see [Hooks](hooks.md). +Channels are validated at compile time (`validateReferences` / send RHS rules) and executed via in-memory queue and dispatch in the Node runtime; durable **`inbox/`** files under the run directory appear only for **routed** sends (audit — see [Inbox & Dispatch](inbox.md)). Hooks are CLI-only: they load from `hooks.json` and run as shell commands with JSON on stdin, driven by the same `__JAIPH_EVENT__` stream as the progress UI — see [Hooks](hooks.md). ## Test runner integration (`*.test.jh` in the kernel) -**How** `jaiph test` wires into the same stack as `jaiph run`: `*.test.jh` files are parsed in the CLI; `runTestFile()` drives blocks in-process. **`buildRuntimeGraph(testFile)`** is called **once per `runTestFile` invocation** and the resulting graph is reused across all blocks and `test_run_workflow` steps (the import closure is constant for a given test file within a single process run). Each `test_run_workflow` step resolves mocks against that cached graph, then constructs `NodeWorkflowRuntime` with `mockBodies` / mock prompt env, passing **`suppressLiveEvents: true`** so the in-process runtime's `__JAIPH_EVENT__` stderr writes are skipped (durable `run_summary.jsonl` writes are unaffected). Without this flag, every workflow event would print to the test process's stderr and swamp `node --test` reporter output. Mock prompts, workflows, rules, and scripts are supported through the runtime's mock infrastructure. +**How** `jaiph test` wires into the same stack as `jaiph run`: `*.test.jh` files are parsed in the CLI; `runTestFile()` drives blocks in-process. **`buildRuntimeGraph(testFile)`** is called **once per `runTestFile` invocation** and the resulting graph is reused across all blocks and `test_run_workflow` steps (the import closure is constant for a given test file within a single process run). Each `test_run_workflow` step resolves mocks against that cached graph, then constructs `NodeWorkflowRuntime` with `mockBodies` / mock prompt env, passing **`suppressLiveEvents: true`** so **`RuntimeEventEmitter`** skips writing **`__JAIPH_EVENT__`** lines to **stderr** while still appending **`run_summary.jsonl`** for that run. Without this flag, every workflow event would print to the test process's stderr and swamp `node --test` reporter output. Mock prompts, workflows, rules, and scripts are supported through the runtime's mock infrastructure. + Before that, the CLI prepares script executables via **`buildScripts(testFileAbs, tmpDir, workspaceRoot)`** — the same **`buildScripts`** helper as `jaiph run`, with the **test file as the entrypoint**. That walks the test module and its **import closure** (transitive `import` edges), runs **`validateReferences`** / **`emitScriptsForModule`** per reachable file, and writes `scripts/` so imported workflows have paths under `JAIPH_SCRIPTS`. Unrelated `*.jh` files elsewhere in the repo are not compiled unless imported. Authoring rules, fixtures, and mock syntax for `*.test.jh` are documented in [Testing](testing.md), not here. ## CLI progress reporting pipeline -Static tree from AST (`progress.ts`); runtime events (`events.ts`, `stderr-handler.ts`); emitter (`emitter.ts`); display (`display.ts`, `progress.ts`). Async branch numbering (subscript ₁₂₃… prefixes) is driven by `async_indices` on step and log events — the runtime propagates a chain of 1-based branch indices through `AsyncLocalStorage`, and the stderr handler renders them at the appropriate indent level. `const` steps whose value is a `match_expr` are walked for nested `run`/`ensure` arms; matched targets appear as child items in the step tree (e.g. `▸ script safe_name` under the `const` row). +The progress UI combines a **static** step tree derived from the workflow AST (`src/cli/run/progress.ts`) with **live** updates from the runtime event stream. Event wiring: `src/cli/run/events.ts` and `src/cli/run/stderr-handler.ts` parse `__JAIPH_EVENT__` lines; `src/cli/run/emitter.ts` bridges into the renderer. Line-oriented formatting (`formatStartLine`, `formatHeartbeatLine`, `formatCompletedLine`) lives primarily in `src/cli/run/display.ts`, which shares some display helpers with `progress.ts`. Async branch numbering (subscript ₁₂₃… prefixes) is driven by `async_indices` on step and log events — the runtime propagates a chain of 1-based branch indices through `AsyncLocalStorage`, and the stderr handler renders them at the appropriate indent level. `const` steps whose value is a `match_expr` are walked for nested `run`/`ensure` arms; matched targets appear as child items in the step tree (e.g. `▸ script safe_name` under the `const` row). This pipeline does not apply to **`jaiph run --raw`**. ## Distribution: Node vs Bun standalone @@ -147,7 +152,7 @@ flowchart TD CLI -->|jaiph run| BS1[buildScripts] BS1 --> Transpile - CLI -->|jaiph test| BS2[buildScripts workspace] + CLI -->|jaiph test| BS2[buildScripts(entry .test.jh)] BS2 --> Transpile BS2 --> TR[Node Test Runner in-process] @@ -166,7 +171,7 @@ flowchart TD TR --> RT RT -->|script steps| SCRIPT[Managed script subprocesses] - RT -->|prompt steps| KERNEL[JS kernel: prompt / emit / inbox / stream / schema / mock] + RT -->|prompt steps| KERNEL[Kernel libs: prompt, events, inbox, stream, schema, mock] RT -->|live events| EV["__JAIPH_EVENT__ stderr only"] EV --> CLI @@ -182,6 +187,8 @@ flowchart TD ## Sequence diagram: regular flow (`*.jh`) +Interactive **`jaiph run`** (no **`--raw`**): banner, progress tree, hooks, and PASS/FAIL footer. + ```mermaid sequenceDiagram participant User @@ -219,6 +226,8 @@ sequenceDiagram CLI-->>User: PASS/FAIL ``` +**Docker:** the inner container command is **`jaiph run --raw …`** (see [Sandboxing](sandboxing.md#docker-container-isolation)): no banner or progress UI inside the container; **`__JAIPH_EVENT__`** lines still appear on stderr for the host CLI to parse. + ## Sequence diagram: `jaiph test` flow ```mermaid @@ -226,7 +235,7 @@ sequenceDiagram participant User participant CLI as CLI jaiph test participant Parser as parsejaiph - participant Prep as buildScripts workspace + participant Prep as buildScripts(test file) participant TestRunner as runTestFile / runTestBlock participant Graph as buildRuntimeGraph participant Runtime as NodeWorkflowRuntime @@ -235,7 +244,7 @@ sequenceDiagram User->>CLI: jaiph test flow.test.jh CLI->>Parser: parse test file Parser-->>CLI: jaiphModule + tests[] blocks - CLI->>Prep: buildScripts(workspace) workspace .jh only + CLI->>Prep: buildScripts(test path, tmp) import closure Prep-->>CLI: scriptsDir CLI->>TestRunner: runTestFile(test path workspace scriptsDir blocks) TestRunner->>Graph: buildRuntimeGraph(test file) once per file @@ -256,8 +265,9 @@ sequenceDiagram ## Summary - `.jh` / `*.test.jh` share parser/AST; **compile-time** validation runs in **`emitScriptsForModule`** during **`buildScripts`**. **`buildRuntimeGraph`** loads modules with **parse-only** imports. -- **`jaiph compile`** walks the same import closures as a normal compile check, runs **`validateReferences`** on each module, and exits — no **`buildScriptFiles`** emission, no **`buildScripts`**, no runner spawn. +- **`jaiph compile`** walks import closures with **`validateReferences` only**, and exits — no **`scripts/`** emission (**no **`buildScriptFiles`** / **`buildScripts`**), no **`buildRuntimeGraph()`**, no runner spawn. Directory discovery omits **`*.test.jh`** unless you pass a test file explicitly. - **Node-only runtime:** all execution — local `jaiph run`, Docker `jaiph run`, and `jaiph test` — goes through `NodeWorkflowRuntime`. Docker containers run `node-workflow-runner` with the compiled JS tree and scripts mounted, using the same semantics as local execution. -- **CLI** owns launch, observation, hooks, and runtime preparation (`buildScripts`). Workflow execution runs in **`NodeWorkflowRuntime`**, with **script steps** as managed subprocesses. +- **CLI** owns launch, observation, hooks (except **`jaiph run --raw`**), and runtime preparation (`buildScripts`). **`jaiph run --raw`** still emits **`__JAIPH_EVENT__`** on stderr from the runtime; the CLI does not attach the interactive progress/hooks pipeline. **`jaiph test`** passes **`suppressLiveEvents: true`** into **`NodeWorkflowRuntime`** so **`RuntimeEventEmitter`** skips writing those live stderr lines while **`run_summary.jsonl`** still records workflow traffic where the emitter appends it. +- Workflow execution runs in **`NodeWorkflowRuntime`**, with **script steps** as managed subprocesses. - No workflow-level `.sh` files or `jaiph_stdlib.sh` are produced or required. - Contracts: `__JAIPH_EVENT__`, `.jaiph/runs`, `run_summary.jsonl`, hook payloads. diff --git a/docs/artifacts.md b/docs/artifacts.md index 23ecab86..397bff1a 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -7,37 +7,44 @@ redirect_from: # Runtime artifacts -Workflow and test runners need two kinds of output: **what humans see right now** (progress, status) and **what is left behind** after the process exits (replay, diffs, CI reports). Jaiph keeps those separate: the **live** channel is `__JAIPH_EVENT__` JSON lines on the child process’s **stderr**; the **durable** side is a tree of files under the project workspace so you can inspect, diff, and archive a run after it finishes. +Long-running orchestration tools usually split **telemetry you watch while something runs** from **evidence you keep after it stops**. The first answers “what is happening now?”; the second answers “what happened, in enough detail to debug or audit later?” Jaiph does the same. -When you run a workflow, or `jaiph test` executes workflows inside test blocks, the **Node** workflow runtime materializes that durable tree. By default it lives at `/.jaiph/runs/`; you can point it elsewhere with `run.logs_dir` / `JAIPH_RUNS_DIR` (see [Configuration — Run keys](configuration.md#run-keys)). The layout below is what `NodeWorkflowRuntime` writes. +For Jaiph, **live** observation is the `__JAIPH_EVENT__` JSON line protocol on the workflow runner’s **stderr** (what the interactive CLI and [Hooks](hooks.md) consume). **Durable** observation is a directory tree on disk: step captures, an append-only summary timeline, optional inbox copies, and a writable `artifacts/` folder for anything workflows publish explicitly. + +When you run a workflow, or `jaiph test` executes workflows inside test blocks, **`NodeWorkflowRuntime`** materializes that durable tree. **`jaiph run`** defaults to `/.jaiph/runs/`; override with `run.logs_dir` or **`JAIPH_RUNS_DIR`** (see [Configuration — Run keys](configuration.md#run-keys)). The test runner uses its own ephemeral runs root under **`JAIPH_RUNS_DIR`** so normal workspace runs are not overwritten — see [Configuration — Testing with `jaiph test`](configuration.md#testing-with-jaiph-test). The layout below matches what the runtime creates in the constructor (see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout)). In Docker mode, paths inside recorded events may use container prefixes (`/jaiph/run/…`); the CLI maps them to host paths when reporting failures — see [Sandboxing — Path remapping](sandboxing.md#path-remapping). ## Run directory layout -The runtime uses a UTC-dated hierarchy. Each run gets its own folder: UTC date, then UTC time plus a basename derived from `JAIPH_SOURCE_FILE` when set, otherwise the entry module’s file basename. +The runtime uses a UTC-dated hierarchy. Each run gets its own folder: UTC date, then UTC time plus a **basename** used only for naming (not a path): **`JAIPH_SOURCE_FILE`** when set in the environment (the CLI and `node-workflow-runner` set this to the entry file basename), otherwise `basename(graph.entryFile)` from the parsed graph. ``` .jaiph/runs/ / # UTC date (see NodeWorkflowRuntime) -/ # UTC time + basename (see above) 000001-module__step.out # stdout capture per step (6-digit seq prefix) - 000001-module__step.err # stderr capture (when non-empty) + 000001-module__step.err # stderr capture (may be empty) artifacts/ # user-published files (`jaiphlang/artifacts`); `JAIPH_ARTIFACTS_DIR` - inbox/ # inbox message files (when channels are used) + inbox/ # audit copies of routed channel payloads (optional) heartbeat # liveness: epoch ms, refreshed about every 10s - return_value.txt # present if `default` workflow exited 0 and returned a value + return_value.txt # `runDefault` only: status 0 and `returnValue` defined (may be "") run_summary.jsonl # durable event timeline (JSON Lines) ``` -Sequence numbers in those filenames are **monotonic and unique** per run: a single in-memory counter in `NodeWorkflowRuntime` increments for each step capture. The separate `seq-alloc` helper is a **file-backed** allocator for tooling; ordinary runs do not use a `.seq` file in the run directory. For the full system picture, see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout) and [Architecture — Contracts](architecture.md#contracts) (`__JAIPH_EVENT__` on stderr is the live path). +Sequence numbers in those filenames are **monotonic and unique** per run. `RuntimeEventEmitter` owns a single in-memory counter (`allocStepSeq`) that advances whenever a step allocates paired capture files: **`executeManagedStep`** (nested **`workflow`** / **`rule`**, **`script`** references, inline scripts, and **`shell`** lines run via `sh -c`) plus **`prompt`** steps (which call `allocStepSeq` inside `emitPromptStepStart`). Ordinary **`log`**, **`logerr`**, **`fail`**, **`send`**, and most **`const`** bindings do **not** open new numbered `.out`/`.err` pairs — they still emit **`LOG`/`LOGERR`** or **`INBOX_ENQUEUE`** records (and related lines) into **`run_summary.jsonl`** where applicable. There is **no** `.seq` file in the run directory. For the live vs durable split, see [Architecture — Contracts](architecture.md#contracts): `__JAIPH_EVENT__` on stderr is the streaming path; `run_summary.jsonl` is the durable timeline. ## What each artifact is for -- **`*.out` / `*.err`** — Per-step capture files for managed work (script subprocesses, nested workflows, rules, and prompt steps). **Stdout** is written to a `.out` file as the step runs; a **`.err` file appears when stderr is non-empty** (see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout)). The live CLI stream is still separate: see [Architecture — Contracts](architecture.md#contracts). -- **`run_summary.jsonl`** — JSON Lines timeline mirroring what also goes to `__JAIPH_EVENT__` (where enabled): workflow boundaries, step start/end, log lines, inbox-related events. The file is created at runtime startup and lines are appended as the run progresses. -- **`inbox/`** — When you use channels, copies of message payloads can appear here for inspection (see [Inbox & Dispatch](inbox.md)). +- **`*.out` / `*.err`** — Paired capture files for steps that record subprocess or prompt I/O. The runtime creates both paths at **`STEP_START`**. For **managed** steps (extracted scripts, nested workflows/rules, single-line `shell`, and similar), stdout/stderr are **streamed** into the files during execution, then **rewritten** with the final aggregated strings at step end — so a long-running step’s `.out` can be tailed while it runs (see [CLI — Run artifacts and live output](cli.md#run-artifacts-and-live-output)). **Prompt** steps stream the model transcript into `.out`; `.err` is only overwritten when stderr from the backend is non-empty (otherwise the placeholder file stays zero-length). **Errors and CLI progress** still use the live `__JAIPH_EVENT__` stream on stderr; these files are the on-disk record. + +- **`run_summary.jsonl`** — Append-only JSON Lines timeline: workflow boundaries, step start/end, `LOG` / `LOGERR`, prompt lifecycle, inbox events, and the same step payload fields as the live stream. It is **truncated to empty at runtime startup**, then each event appends a line via `appendRunSummaryLine` as execution proceeds. The in-process test runner can set `suppressLiveEvents`, which **stops** `__JAIPH_EVENT__` lines from going to stderr while **`run_summary.jsonl` keeps updating** (see [Architecture — Core components](architecture.md#core-components), `RuntimeEventEmitter`). + +- **`inbox/`** — When channels are used, a **`send`** may persist a copy of the payload here (`NNN-.txt`) for audit. The runtime walks ancestor workflow contexts and writes a file **only when it finds a matching route for that channel** on the stack (same condition as “routed” dispatch — see [Inbox & Dispatch](inbox.md)); unrouted sends enqueue without creating `inbox/` files. Delivery stays in-memory; this directory is not a mailbox API. + - **`heartbeat`** — Best-effort file containing a wall-clock millisecond timestamp, rewritten on a timer (~10s). Liveness for external watchdogs; not required for normal CLI use. -- **`return_value.txt`** — Written after a successful `default` workflow when the workflow returns a value (including empty string, which yields a zero-length file so it is distinct from “no return”). Other entry paths (e.g. `test_run_workflow`) are not required to create this file. -- **`artifacts/`** — The runtime creates this directory in the run folder before execution and sets `JAIPH_ARTIFACTS_DIR` to it (along with `JAIPH_RUN_DIR`, `JAIPH_RUN_ID`, and `JAIPH_RUN_SUMMARY_FILE`). User code typically writes here via the `jaiphlang/artifacts` library (`artifacts.save`). In Docker mode this directory is under the **host-writable** run mount (`/jaiph/run/...` in the container), not the read-only workspace overlay. See [Libraries — `jaiphlang/artifacts`](libraries.md#jaiphlangartifacts--publishing-files-out-of-the-sandbox) and [Sandboxing](sandboxing.md). + +- **`return_value.txt`** — Written only from **`runDefault`** (the normal **`jaiph run`** entry path) when the top-level workflow finishes with **exit status 0** and the aggregated result has **`returnValue !== undefined`** (empty string is allowed and produces a zero-byte file; **`undefined`** means the file is omitted — typically “fell off the end” of the workflow without a **`return`**). **`runNamedWorkflow`** (`test_run_workflow`, nested named runs, etc.) returns the value to the caller but does **not** write this file. + +- **`artifacts/`** — Created in the constructor together with the empty **`run_summary.jsonl`** (truncated file). The runtime sets **`JAIPH_ARTIFACTS_DIR`**, **`JAIPH_RUN_DIR`**, **`JAIPH_RUN_SUMMARY_FILE`**, and **`JAIPH_RUN_ID`**: if **`JAIPH_RUN_ID`** is already set in the incoming environment it is preserved; otherwise a new UUID is generated. User workflows usually publish into **`artifacts/`** through **`jaiphlang/artifacts`** (`artifacts.save`). In Docker mode it sits under the **host-writable** run mount (`/jaiph/run/...` inside the container), not the read-only workspace overlay. See [Libraries — `jaiphlang/artifacts`](libraries.md#jaiphlangartifacts--publishing-files-out-of-the-sandbox) and [Sandboxing](sandboxing.md). ## Keeping runs out of git diff --git a/docs/cli.md b/docs/cli.md index ddd7c6a0..b956872f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -9,15 +9,21 @@ redirect_from: Jaiph is a workflow system: authors write `.jh` modules, and a **TypeScript CLI** prepares scripts, launches a **Node workflow runtime**, and surfaces progress while the **JavaScript kernel** executes the AST in process (no separate workflow shell). The CLI is what you install as the `jaiph` binary — it is the boundary between your terminal or CI and the interpreter. -This page lists **commands**, important **flags**, and **environment variables**. It focuses on how the tool behaves, not on the language itself. For syntax and step semantics, see [Grammar](grammar.md). For repository layout, pipelines, and contracts (`__JAIPH_EVENT__`, artifacts, Docker vs local), see [Architecture](architecture.md). +At a high level, the CLI does four things: **compile** script bodies from your module graph (`buildScripts`), **spawn** the detached workflow runner (`node-workflow-runner`) for `jaiph run`, **observe** `__JAIPH_EVENT__` lines on stderr to render progress and drive hooks (unless `--raw`), and **leave** durable artifacts under `.jaiph/runs`. `jaiph test` reuses the same compilation step and runtime kernel but executes test blocks in-process with mocks — see [Architecture](architecture.md) for the full pipeline. + +This page lists **commands**, important **flags**, and **environment variables**. It focuses on how the tool behaves, not on the language itself. For semantics and the overall language model, see [Language](language.md). For concrete syntax rules (imports, orchestration strings, managed calls, …), see [Grammar](grammar.md). For repository layout, pipelines, and contracts (`__JAIPH_EVENT__`, artifacts, Docker vs local), see [Architecture](architecture.md). **Commands:** `run`, `test`, `compile`, `format`, `init`, `install`, `use`. -**Global options:** `-h` / `--help` and `-v` / `--version` are recognized only as the **first token after `jaiph`** (e.g. `jaiph --help`). They are not treated as global flags after a subcommand or a file path (`jaiph run --help` is **not** usage — use `jaiph --help`, or `jaiph compile -h` for compile-specific usage). +**Global options:** `-h` / `--help` and `-v` / `--version` are recognized only as the **first token after `jaiph`** (e.g. `jaiph --help`). They are not treated as global flags after a subcommand or a file path (`jaiph run --help` is **not** usage — use `jaiph --help`, or **`jaiph compile -h`** / **`jaiph compile --help`** for compile-specific usage — the `compile` command parses `-h` / `--help` after the subcommand). Running **`jaiph`** with no arguments prints the same overview and exits **0**. + +Any other unknown first token prints `Unknown command: …`, repeats the overview, and exits **1**. ## File shorthand -If the first argument is an existing file, Jaiph routes it automatically based on the extension. Files ending in **`*.test.jh`** are run as tests (same as `jaiph test `). Other files ending in **`*.jh`** are run as workflows (same as `jaiph run `). The `*.test.jh` check happens first, so test files are never mistaken for workflows. +If the **first argument after `jaiph`** is an **existing path** (resolved relative to the current working directory), Jaiph routes it automatically based on the extension. Files ending in **`*.test.jh`** are run as tests (same as `jaiph test `). Other paths ending in **`.jh`** are run as workflows (same as `jaiph run `). The `*.test.jh` check happens first, so test modules are never mistaken for workflows. Paths that do not exist fall through to normal command parsing (e.g. you cannot rely on shorthand for a not-yet-created file). + +Additional positional tokens after a **workflow** shorthand are forwarded to **`workflow default`**, matching `jaiph run`. Tokens after a **test** shorthand are accepted but **ignored** (same as `jaiph test ` with extra arguments). ```bash # Workflow shorthand @@ -45,7 +51,7 @@ Any path ending in `.jh` is accepted (including `*.test.jh`, since the extension **Flags:** - **`--target `** — keep emitted script files and run metadata under `` instead of a temp directory (useful for debugging). -- **`--raw`** — skip the banner, live progress tree, hooks, and CLI failure footer. The workflow runner child uses **inherited stdio** so `__JAIPH_EVENT__` JSON lines go to **stderr** unchanged. The **host** CLI relies on this for Docker-backed runs (the container invokes `jaiph run --raw` so the host parses events from Docker’s stderr); you can also use it when embedding Jaiph in another tool. See [Sandboxing — Runtime behavior](sandboxing.md#runtime-behavior). +- **`--raw`** — skip the banner, live progress tree, hooks, and CLI failure footer. The workflow runner child uses **inherited stdio** so `__JAIPH_EVENT__` JSON lines go to **stderr** unchanged. When **Docker sandboxing** is used, the **host** runs interactive `jaiph run` and the **container** runs `jaiph run --raw …` so the host can parse events from the container’s stderr ([Architecture](architecture.md), [Sandboxing](sandboxing.md)). **Important:** if you invoke `jaiph run --raw` yourself on the host, the CLI takes a separate code path that **never starts Docker** — workflow execution runs locally in that process even when `JAIPH_DOCKER_ENABLED=true`. Use `--raw` for embedding or piping; use interactive `jaiph run` (no `--raw`) when you want the CLI to apply sandbox env rules. There is no PASS/FAIL line, **`return_value.txt` is not printed to stdout**, and the process exit code alone reflects success or failure. See [Sandboxing — Runtime behavior](sandboxing.md#runtime-behavior). - **`--`** — end of Jaiph flags; remaining args are passed to `workflow default` (e.g. `jaiph run file.jh -- --verbose`). **Examples:** @@ -73,7 +79,7 @@ workflow default() { } ``` -Workflow and rule bodies contain structured Jaiph steps only — use `run` to call a `script` for shell execution. In bash-bearing contexts (mainly `script` bodies, and restricted `const` / send RHS forms), `$(...)` and the first command word are validated: they must not invoke Jaiph rules, workflows, or scripts, contain inbox send (`<-`), or use `run` / `ensure` as shell commands (`E_VALIDATE`). See [Grammar — Managed calls vs command substitution](grammar.md#managed-calls-vs-command-substitution). +**Rule** bodies are **managed steps only** — no raw shell lines; use `run` to a `script` for shell execution. **Workflow** bodies may include **inline shell** lines that do not parse as a Jaiph step (the compiler still validates them); for anything non-trivial, prefer a top-level `script` and `run`. In bash-bearing contexts (mainly `script` bodies, and restricted `const` / send RHS forms), `$(...)` and the first command word are validated: they must not invoke Jaiph rules, workflows, or scripts, contain inbox send (`<-`), or use `run` / `ensure` as shell commands (`E_VALIDATE`). See [Grammar — Language concepts](grammar.md#language-concepts) and [Grammar — Managed calls vs command substitution](grammar.md#managed-calls-vs-command-substitution). For `const` in those bodies, a reference plus arguments on the RHS must be written as `const name = run ref([args...])` (or `ensure` for rule capture), not as `const name = ref([args...])` — the latter is `E_PARSE` with text that explains the fix. @@ -107,6 +113,8 @@ The root PASS/FAIL summary uses the format `✓ PASS workflow default (0.2s)`. C **TTY mode:** one extra line at the bottom shows the running workflow and elapsed time: `▸ RUNNING workflow (X.Xs)` — updated in place every second. When the run completes, it is replaced by the final PASS/FAIL line. +**Successful exit:** when the default workflow exits **0**, the CLI prints `✓ PASS workflow default (...)` plus elapsed time (see above). If the workflow **returns** a value, the runtime writes `return_value.txt` under the run directory; the CLI prints that value on stdout **after** the PASS line, separated by a blank line (host paths are unchanged; Docker runs remap container paths when reading the file). See [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout). + **Non-TTY mode** (CI, pipes, log capture): no RUNNING line and no in-place updates. Step start (▸) and completion (✓/✗) lines still print as they occur. Long-running steps additionally print **heartbeat** lines to avoid looking like a hang: - Format: `· (running s)` — entire line dim/gray (plain text with `NO_COLOR`). @@ -169,7 +177,7 @@ In Docker mode, artifact paths recorded by the container use container-internal ### Run artifacts and live output -Each run directory is `//-/`, where date and time are UTC and `` is `JAIPH_SOURCE_FILE` if set, otherwise the entry file basename. Each step gets sequenced capture files: `000001-module__rule.out` for stdout, and `000002-module__workflow.err` for stderr **when that stream is non-empty** (see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout)). +Each run directory is `//-/`, where date and time are UTC and `` is `JAIPH_SOURCE_FILE` if set, otherwise the entry file basename. Steps that allocate captures open **paired** `NNNNNN-.out` and `.err` files at **`STEP_START`** (see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout) and [Runtime artifacts — What each artifact is for](artifacts.md#what-each-artifact-is-for)). Step **stdout** artifacts are written **incrementally during execution**, so you can tail a running step's output in real time: @@ -181,7 +189,7 @@ jaiph run ./flows/deploy.jh tail -f .jaiph/runs/2026-03-22/14-30-00-deploy.jh/000003-deploy__run_migrations.out ``` -If a stream stays empty for a step, the runtime may omit that artifact file. Any empty capture files are cleaned up at step end. +Which steps get numbered `.out`/`.err` pairs, how prompts differ from managed scripts, and when empty files are removed are spelled out in [Runtime artifacts](artifacts.md); the durable timeline either way is **`run_summary.jsonl`**. ### Run summary (`run_summary.jsonl`) {#run-summary-jsonl} @@ -247,9 +255,9 @@ The test runner uses the same Node workflow runtime as `jaiph run`. For each tes **Usage:** -- `jaiph test` — discover and run all `*.test.jh` under the workspace root. The workspace root is found by walking up from the current directory until a directory with `.jaiph` or `.git` is found; if neither exists, the current directory is used. -- `jaiph test ` — run all `*.test.jh` files recursively under the given directory. -- `jaiph test ` — run a single test file. +- `jaiph test` — discover and run all `*.test.jh` under the workspace root. The workspace root is found by walking up from the **current working directory** until a directory with `.jaiph` or `.git` is found; if neither exists, the current directory is used (same `detectWorkspaceRoot` algorithm as `jaiph run` / `jaiph install`). +- `jaiph test ` — run all `*.test.jh` files recursively under the given directory. Workspace root for script compilation is detected by walking up from **that directory** (resolved), not necessarily from your shell cwd. +- `jaiph test ` — run a single test file; workspace root is detected from the test file’s directory. With no arguments, or with a directory that contains no test files, the command exits with status **1** and prints an error. @@ -272,16 +280,16 @@ Parse modules and run **`validateReferences`** (the same compile-time checks as jaiph compile [--json] [--workspace ] ... ``` -At least one path is required. +At least one path is required. **`jaiph compile -h`** or **`jaiph compile --help`** prints command-specific usage and exits **0**. **File arguments** — Each `*.jh` file is expanded to its **transitive import closure**; every module in the union is parsed and validated once. -**Directory arguments** — The tree is scanned for `*.jh` files whose basename is **not** `*.test.jh`; each such file is treated as an entrypoint and its closure merged into the same validation set. To validate a test module’s graph explicitly, pass that **`*.test.jh` file** as a path (directories never pick up `*.test.jh` as roots). +**Directory arguments** — The tree is scanned for `*.jh` files whose basename is **not** `*.test.jh` (same rule as `walkjhFiles` in the transpiler: files like `foo.test.jh` are skipped). Each non-test `*.jh` under the tree is treated as an entrypoint and its closure merged into the same validation set. To validate a test module’s graph explicitly, pass that **`*.test.jh` file** as a path (directories never pick up `*.test.jh` as roots). **Flags:** - **`--json`** — On success, print `[]` to stdout. On failure, print one JSON **array** of objects `{ "file", "line", "col", "code", "message" }` to stdout and exit **1** (non-JSON errors use a synthetic `E_COMPILE` object when the message is not in `file:line:col CODE …` form). -- **`--workspace `** — Override the workspace root used for **library import resolution** (`/.jaiph/libs/`, etc.) for all derived paths. When omitted, the workspace is auto-detected per file the same way as `jaiph run`. +- **`--workspace `** — Override the workspace root used for **library import resolution** (`/.jaiph/libs/`, etc.) for **all** modules reached from the given paths. When omitted, the workspace is **auto-detected** from each path’s location (`detectWorkspaceRoot` — same algorithm as `jaiph run`, starting from the file’s directory or from a directory argument). ## `jaiph format` @@ -308,7 +316,7 @@ One or more file paths are required (each path must end with `.jh`, e.g. `flow.j # Rewrite files in place jaiph format flow.jh utils.jh -# Check formatting in CI (non-zero exit on drift) +# Check formatting in CI (non-zero exit on drift); ensure globs expand to real paths jaiph format --check src/**/*.jh # Use 4-space indentation @@ -327,22 +335,22 @@ Creates: - `.jaiph/.gitignore` — lists `runs` and `tmp`. If the file already exists and does not match this exact list, `jaiph init` exits with a non-zero status. - `.jaiph/bootstrap.jh` — canonical bootstrap workflow; made executable. The template uses a triple-quoted multiline prompt body (`prompt """ ... """`) so the generated file parses and compiles as valid Jaiph. It asks the agent to scaffold workflows under `.jaiph/` and ends by logging a summary (`WHAT CHANGED` + `WHY`). Docker sandboxing uses the default `ghcr.io/jaiphlang/jaiph-runtime` image unless you set `runtime.docker_image` or `JAIPH_DOCKER_IMAGE`. -- `.jaiph/SKILL.md` — copied from the skill file bundled with your Jaiph installation (or from `JAIPH_SKILL_PATH` when set). If no skill file is found, this file is not written and a note is printed. +- `.jaiph/SKILL.md` — copied when the CLI can resolve a skill markdown file: if **`JAIPH_SKILL_PATH`** is set **and** that path exists, it wins; otherwise the CLI tries install-relative paths (`jaiph-skill.md` beside the packaged tree, then `docs/jaiph-skill.md` beside the package), then **`docs/jaiph-skill.md` under the current working directory**. If none of these exist, `SKILL.md` is not written and a note is printed. ## `jaiph install` -Install project-scoped libraries. Libraries are git repos cloned into `.jaiph/libs//` under the workspace root. A lockfile (`.jaiph/libs.lock`) tracks installed libraries for reproducible setups. +Install project-scoped libraries. Libraries are git repos cloned into `.jaiph/libs//` under the **workspace root**. The workspace is determined from the **current working directory** (`detectWorkspaceRoot(process.cwd())` — walk upward until `.jaiph` or `.git`, with the same temp-directory guards as `jaiph run`). A lockfile (`.jaiph/libs.lock`) under that root tracks installed libraries for reproducible setups. ```bash jaiph install [--force] ... jaiph install [--force] ``` -**With arguments** — clone each repo into `.jaiph/libs//` (shallow: `--depth 1`) and upsert the entry in `.jaiph/libs.lock`. The library name is derived from the URL: last path segment, stripped of `.git` suffix (e.g. `github.com/you/queue-lib.git` → `queue-lib`). Version pinning uses `@` after the URL. +**With arguments** — clone each repo into `.jaiph/libs//` (shallow: `--depth 1`) and upsert the entry in `.jaiph/libs.lock`. The library name is derived from the URL: last path segment, stripped of `.git` suffix (e.g. `github.com/you/queue-lib.git` → `queue-lib`). Version pinning is usually written as **`https://…/name.git@`**; other URL shapes with a trailing **`@ref`** are also accepted when the parser can split URL and version unambiguously. -**Without arguments** — restore all libraries from `.jaiph/libs.lock`. Useful after cloning a project or in CI. +**Without arguments** — restore all libraries from `.jaiph/libs.lock`. Useful after cloning a project or in CI. If the lockfile exists but lists **no** libraries, the command prints `No libs in lockfile.` and exits **0**. -If `.jaiph/libs//` already exists, the library is skipped. Use `--force` to delete and re-clone. +If `.jaiph/libs//` already exists, the library is skipped. Use **`--force`** (anywhere in the argument list) to delete and re-clone. **Lockfile format** (`.jaiph/libs.lock`): @@ -393,7 +401,7 @@ jaiph use ```bash jaiph use nightly -jaiph use 0.9.3 +jaiph use 0.9.4 ``` ## File extension @@ -408,14 +416,16 @@ These variables apply to `jaiph run` and workflow execution. Variables marked ** **Internal variables:** -- `JAIPH_META_FILE` — path to the metadata file the CLI writes under the build output directory; the workflow runner reads it after exit. Set by the launcher on the child process; `resolveRuntimeEnv` removes any inherited value from the parent. -- `JAIPH_RUN_DIR`, `JAIPH_RUN_ID`, `JAIPH_RUN_SUMMARY_FILE` — `JAIPH_RUN_ID` is generated by the host CLI as a UUID per `jaiph run` invocation and forwarded to the runtime (and into the Docker container when sandboxed). The runtime uses this value as the workflow run identifier; if unset, the runtime generates its own UUID. `JAIPH_RUN_DIR` and `JAIPH_RUN_SUMMARY_FILE` are set by `NodeWorkflowRuntime` to the run directory and `run_summary.jsonl` path. -- `JAIPH_SOURCE_FILE` — set automatically by the CLI to the entry file basename. Used to name run directories. +- `JAIPH_META_FILE` — path to the run metadata file (under the CLI’s build output directory for that invocation). Set on the **detached workflow child** only; the parent strips any inherited value so leftover exports do not collide. The runner writes `run_dir=` / `summary_file=` lines for the host to read after exit. +- `JAIPH_SOURCE_ABS` — absolute path to the entry `.jh` file; set by the CLI for **`jaiph run`** before spawn. Required by the runner (local and Docker). +- `JAIPH_SCRIPTS` — directory containing emitted **`script`** files for this run; set after **`buildScripts()`**. Any **`JAIPH_SCRIPTS`** exported in the parent shell is cleared before launch so nested toolchains do not point at the wrong tree. +- `JAIPH_RUN_DIR`, `JAIPH_RUN_ID`, `JAIPH_RUN_SUMMARY_FILE` — for a normal (**non-raw**) **`jaiph run`**, the host generates **`JAIPH_RUN_ID`** once per invocation (UUID), passes it through to the detached child (and into Docker when sandboxed), and Docker failure-path discovery can match summaries by this id. The runtime uses **`JAIPH_RUN_ID`** as the stable run identifier; if it is absent, the runtime may assign its own UUID. **`JAIPH_RUN_DIR`** and **`JAIPH_RUN_SUMMARY_FILE`** are set inside the runner once the UTC run directory exists. +- `JAIPH_SOURCE_FILE` — set automatically by the CLI to the entry file **basename**. Used to name run directories (see [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout)). **Workspace and run paths:** - `JAIPH_WORKSPACE` — workspace root, set by the CLI. Detected by walking up from the entry `.jh` file's directory until `.jaiph` or `.git` is found. Guards in `detectWorkspaceRoot` skip misleading markers under shared system temp directories (`/tmp`, `/var/tmp`, macOS `/var/folders/.../T/...`) and nested `.jaiph/tmp` trees. In Docker sandbox mode the runtime remaps it inside the container (see [Sandboxing](sandboxing.md)). -- `JAIPH_RUNS_DIR` — root directory for run logs (default: `.jaiph/runs` under workspace). +- `JAIPH_RUNS_DIR` — root directory for run logs. If unset in the environment, the CLI merges the entry module **`config`** field **`run.logs_dir`** (when present) into the spawned process environment; otherwise the default layout is `.jaiph/runs` under the workspace. Exporting **`JAIPH_RUNS_DIR` yourself locks that choice: in-file **`run.logs_dir`** cannot override an environment-provided value. **Agent and prompt configuration:** @@ -461,4 +471,4 @@ For overlay vs copy workspace mode, mounts, and stderr wiring, see [Sandboxing]( ### `jaiph init` -- `JAIPH_SKILL_PATH` — path to the skill markdown copied to `.jaiph/SKILL.md` when running `jaiph init`. +- `JAIPH_SKILL_PATH` — path to the skill markdown copied to `.jaiph/SKILL.md` when running `jaiph init`. The file **must exist** at this path; otherwise the variable is ignored and the CLI falls back to the same install-relative and `docs/jaiph-skill.md` (cwd) search described under [`jaiph init`](#jaiph-init). diff --git a/docs/configuration.md b/docs/configuration.md index 85dbf640..e433b8f7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ When you need the same workflow sources to behave differently on different machi All execution is interpreted by the Node workflow runtime (`NodeWorkflowRuntime`): the AST, managed scripts, prompts, channels, inbox, and `.jaiph/runs` artifacts (see [Architecture](architecture.md)). Configuration only adjusts that stack; it does not change the workflow language or the compile graph. -`jaiph compile` and `buildScripts()` use the same parser, so **unknown `config` keys and wrong value types** fail with deterministic parse errors. Runtime graph loading is parse-only; **compile-time** validation of references runs in the transpile path, not in `buildRuntimeGraph()` (see [Architecture — Summary](architecture.md#summary)). +`jaiph compile` parses each module in the import closure (same grammar as `emitScriptsForModule`), so **unknown `config` keys and wrong value types** surface as the same parse diagnostics as before `jaiph run`. With a **directory** argument it treats every non-test **`*.jh`** file in that directory as its own entrypoint (see `walkjhFiles` — `*.test.jh` is skipped unless you pass a test file explicitly) and validates each entry’s transitive imports. **`validateReferences` only** — no `scripts/` emission, no `buildRuntimeGraph()`, no runner spawn (see [Architecture](architecture.md#summary)). Runtime graph loading is parse-only; **compile-time** reference validation runs in the transpile path, not in `buildRuntimeGraph()`. **Source of truth:** When this document and the implementation disagree, treat the source code as authoritative. @@ -27,7 +27,9 @@ For **agent and run keys**, the full precedence chain is: > **environment > workflow-level config > module-level config > defaults** -For **`runtime.*` (image, network, timeout)**, the CLI merges at **`jaiph run` launch** — not inside `NodeWorkflowRuntime` — in the order **`JAIPH_DOCKER_*` environment > in-file `runtime.*` > defaults** (and separately: Docker on/off is env-only, see above and [Precedence in detail](#precedence-in-detail)). `runtime.*` cannot appear in workflow-level `config` blocks. +`run.recover_limit` is an exception: only **module-level** values affect `run … recover` (see [Run keys](#run-keys)). + +For **`runtime.*` (image, network, timeout)**, the host CLI merges them when it **may spawn Docker** (`resolveDockerConfig` in `src/runtime/docker.ts`) — not inside `NodeWorkflowRuntime`. Precedence is **`JAIPH_DOCKER_*` environment > module-level `runtime.*` > defaults** (Docker on/off remains env-only, see above and [Precedence in detail](#precedence-in-detail)). A **host** invocation of **`jaiph run --raw`** skips that driver entirely and always runs the workflow runner **locally** (no container); **`runtime.*` is unused on that path**. Sandboxed workflows still run `jaiph run --raw …` **inside** the container. `runtime.*` cannot appear in workflow-level `config` blocks. ## In-file config blocks @@ -98,7 +100,8 @@ workflow default() { - At most one per workflow; it must be the first non-comment construct in the body. A duplicate is `E_PARSE`: `duplicate config block inside workflow (only one allowed per workflow)`. - Only **`agent.*` and `run.*` keys** are allowed. Any `runtime.*` or `module.*` key is `E_PARSE`. -- Workflow-level values apply to all steps in that workflow, including `ensure`d rules and scripts called from it. When the workflow finishes, the previous environment is restored. +- Workflow-level values apply to all steps in that workflow, including `ensure`d rules and scripts called from it, for **`agent.*`** and **`run.logs_dir`** / **`run.debug`** (merged when the workflow or cross-module `ensure` runs). **`run.recover_limit` is different:** the retry limit for `run … recover` comes only from the **module-level** `config` of the **`.jh` file that owns the current scope** when the step runs; a workflow-level `run.recover_limit` assignment is valid syntax but does **not** change recover behavior today. +- When the workflow finishes, the previous environment is restored. **Sibling isolation:** Each workflow gets its own clone of the parent environment. Sibling workflows never see each other's config — even when they execute sequentially. If workflow `alpha` sets `agent.backend = "claude"` and workflow `beta` only sets `agent.default_model = "beta-model"`, `beta` still sees the module-level backend (e.g. `"cursor"`), not `alpha`'s. @@ -134,7 +137,7 @@ These control runtime behavior unrelated to the agent. |-----|------|---------|--------------|-------------| | `run.logs_dir` | string | `.jaiph/runs` | `JAIPH_RUNS_DIR` | Step log directory. Relative paths are joined with the workspace root; absolute paths are used as-is. | | `run.debug` | boolean | `false` | `JAIPH_DEBUG` | Enables debug tracing for the run. | -| `run.recover_limit` | integer | `10` | _(no env override)_ | Maximum number of retry attempts for `run … recover` loops before the step fails. See [Language — `recover`](language.md#recover--repair-and-retry-loop). | +| `run.recover_limit` | integer | `10` | _(no env override)_ | Maximum attempts for `run … recover` loops before the step fails (see [Language — `recover`](language.md#recover--repair-and-retry-loop)). Effective value comes **only** from the **module-level** `config` block of the **`.jh` file that owns the current scope** (the file containing the workflow or rule that executes the step). Workflow-level `run.recover_limit` does not apply. | ### Module keys @@ -163,10 +166,12 @@ workflow default() { ### Runtime keys (Docker sandbox — beta) -These configure Docker sandboxing. Unlike agent and run keys, runtime keys are resolved by the `jaiph run` CLI at launch — not by the workflow runtime. They can only appear in **module-level** config blocks (not workflow-level). +These configure Docker sandboxing. Unlike agent and run keys, they are read when the CLI considers a **Docker launch** for interactive **`jaiph run`** (`src/cli/commands/run.ts` → `spawnExec`). They never affect **`NodeWorkflowRuntime`** directly. They can only appear in **module-level** config blocks (not workflow-level). > Docker sandboxing is in **beta**. See [Sandboxing](sandboxing.md) for mounts, workspace layout, env forwarding, path remapping, and container behavior. +> **Host `--raw`:** If you run **`jaiph run --raw`** yourself on the host, the CLI does not enter the Docker branch; image/network/timeout merge is irrelevant for that invocation. Embedding and container flows use **`--raw` inside** the sandbox where the CLI has already picked the image — see [Architecture](architecture.md#sequence-diagram-regular-flow-jh). + | Key | Type | Default | Env variable | Description | |-----|------|---------|--------------|-------------| | `runtime.docker_image` | string | `ghcr.io/jaiphlang/jaiph-runtime:` | `JAIPH_DOCKER_IMAGE` | Image name. Must already contain `jaiph`. When unset, uses the official GHCR image tag matching the installed jaiph version. For a custom image, build and push (or tag locally), then set this key or `JAIPH_DOCKER_IMAGE`. | @@ -184,7 +189,7 @@ For **agent and run keys**, resolution order (highest wins): 3. **Module-level `config`** — applies to workflows that don't define their own block. 4. **Built-in defaults.** -For **Docker enablement**, the `jaiph run` driver uses **`JAIPH_DOCKER_ENABLED` env > unsafe default rule** (env only; `runtime.docker_enabled` is no longer supported). The default rule enables Docker unless `JAIPH_UNSAFE=true` is set; `CI=true` no longer disables Docker (see [Sandboxing — Enabling Docker](sandboxing.md#enabling-docker)). For other `runtime.*` keys (image, network, timeout), the merge is **`JAIPH_DOCKER_*` env > module-level `runtime.*` > defaults**. Workflow-level config cannot set runtime keys. +For **Docker enablement** on **interactive** **`jaiph run`** (no `--raw` on the host), the CLI uses **`JAIPH_DOCKER_ENABLED` env > unsafe default rule** (env only; `runtime.docker_enabled` is no longer supported). The default rule enables Docker unless `JAIPH_UNSAFE=true` is set; `CI=true` no longer disables Docker (see [Sandboxing — Enabling Docker](sandboxing.md#enabling-docker)). **Host** **`jaiph run --raw`** never consults this branch. For other `runtime.*` keys (image, network, timeout), the merge is **`JAIPH_DOCKER_*` env > module-level `runtime.*` > defaults** whenever Docker launch is considered. Workflow-level config cannot set runtime keys. ### Locked variables @@ -224,7 +229,9 @@ Backend-specific flags come from `agent.cursor_flags` / `agent.claude_flags` (or ### Custom agent commands -When `agent.command` points to an executable other than `cursor-agent`, Jaiph treats it as a **custom agent command**. This lets you use any shell script, Python wrapper, or CLI tool as a prompt backend — no need to implement the `stream-json` protocol. +Only the **cursor** backend consults **`agent.command`**. For **`claude`** and **`codex`**, Jaiph always invokes the Claude CLI or the Codex HTTP path (`prompt.ts`), regardless of `agent.command`. + +When **`agent.backend` is `cursor`** (the default) and `agent.command`’s basename is anything other than `cursor-agent`, Jaiph treats it as a **custom agent command**. That lets you use a shell script, Python wrapper, or other CLI as a prompt backend — no need to implement the `stream-json` protocol. **How it works:** @@ -289,15 +296,15 @@ When a `prompt` step runs, Jaiph resolves the effective model using this order: `agent.default_model` applies to **cursor**, **claude**, and **codex**. For the **Claude** backend, when `agent.default_model` is set and `agent.claude_flags` does not already contain `--model`, Jaiph passes `--model ` to the Claude CLI automatically. If both are set, the value in `agent.claude_flags` takes precedence (it is appended last). -**Diagnostics.** Every prompt step records the resolved model in `PROMPT_START` and `PROMPT_END` events in `run_summary.jsonl`: +**Diagnostics.** Every prompt step records model metadata in **`PROMPT_START`** and **`PROMPT_END`** in **`run_summary.jsonl`** (`model`, `model_reason`): ```jsonl {"type":"PROMPT_START","backend":"cursor","model":"gpt-4","model_reason":"explicit",...} ``` -The `model_reason` field is one of: `explicit` (from `agent.default_model`), `flags` (extracted from backend flags), or `backend-default` (no model configured — the backend picks its own). Inspect these events directly in the run summary file. +`model_reason` is one of: **`explicit`** (non-empty **`agent.default_model` / `JAIPH_AGENT_MODEL`**), **`flags`** (`--model` taken from **`agent.cursor_flags`** or **`agent.claude_flags`**), or **`backend-default`** (no resolved model string — Cursor/Claude binaries choose their own; **codex** also reports this when no model is configured, **even though** the HTTP client defaults to **`gpt-4o`**, so the **`model`** field may be omitted there). Inspect these events directly in the summary file. -**No-model troubleshooting.** If the backend rejects the auto-selected default, set `agent.default_model` explicitly or pass `--model ` in the backend-specific flags. +**No-model troubleshooting.** If the backend rejects the auto-selected default, set **`agent.default_model`** (all backends). For **cursor** and **claude** you can also pass **`--model `** in **`agent.cursor_flags`** / **`agent.claude_flags`**; **codex** has no flag channel — use **`agent.default_model`** or env **`JAIPH_AGENT_MODEL`** only. ## Testing with `jaiph test` @@ -327,6 +334,13 @@ Quick reference for all in-file keys and their environment variable equivalents: | `module.version` | _(no env override)_ | | `module.description` | _(no env override)_ | +There is **no in-file key** for the Codex HTTP endpoint or API key. Use environment only: + +| Purpose | Environment variable | +|---------|----------------------| +| OpenAI-compatible API key (required for **codex**) | `OPENAI_API_KEY` | +| OpenAI-compatible chat-completions URL override | `JAIPH_CODEX_API_URL` | + ## Inspecting effective config at runtime Inside workflows, rules, and scripts, agent and run settings are visible as `JAIPH_*` environment variables. In orchestration strings, `${IDENTIFIER}` resolves from workflow variables first, then from the process environment. @@ -343,4 +357,4 @@ The runtime also sets `JAIPH_ARTIFACTS_DIR` — the absolute path to the writabl ## Created by `jaiph init` -`jaiph init` creates `.jaiph/bootstrap.jh` and writes `.jaiph/SKILL.md` from the skill file bundled with your installation (see `JAIPH_SKILL_PATH` in the CLI reference). It does not add a separate config file — use `config { ... }` in your workflow sources. +`jaiph init` creates `.jaiph/bootstrap.jh`, writes `.jaiph/SKILL.md` from the skill file bundled with your installation (see `JAIPH_SKILL_PATH` in the [CLI](cli.md) reference), and ensures `.jaiph/.gitignore` matches the canonical template (lists `runs` and `tmp` under `.jaiph/`). It does not add a separate config file — use `config { ... }` in your workflow sources. diff --git a/docs/contributing.md b/docs/contributing.md index a53f3fac..fbbd1422 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -7,7 +7,11 @@ redirect_from: # Contributing to Jaiph -A shared workflow needs shared expectations: which branch to target, how to build from a clone, and what evidence a change should carry. **This page is that contract for Jaiph** — branching, local install, code and testing philosophy, the layered test stack (TypeScript, txtar, goldens, bash E2E), and what CI enforces. It does **not** teach the language; for that, use [Getting Started](getting-started.md) (documentation map), [Setup](setup.md) (install and workspace), and [Grammar](grammar.md). For **how the implementation is structured** (components, compile and run pipelines, `buildRuntimeGraph` vs validation, runtime contracts, artifact paths), use [Architecture](architecture.md) as the source of truth. +Contributor docs answer a narrow question: **where changes belong**, **how to run the same checks CI runs**, and **which test layer** should encode a behavior change. + +At a high level, Jaiph is built as described in [Architecture](architecture.md) — transpile path (`emitScriptsForModule`, `buildScripts`), parse-only **`buildRuntimeGraph()`**, **`jaiph compile`** (validate-only), **`NodeWorkflowRuntime`**, artifact layout, and Docker helper contracts. Treat that page as authoritative for pipelines and boundaries; if anything here diverges from it or from the implementation, prefer **architecture + source**. + +For workflow syntax, library usage, tooling setup, and grammar details, see [Language](language.md), [Setup](setup.md), [Grammar](grammar.md), and the overview in [Getting Started](getting-started.md). ## Branching and pull requests @@ -39,7 +43,7 @@ The script installs from local source (including uncommitted changes) and places For day-to-day work on the compiler and CLI you usually stay inside the clone: install dev dependencies once, then build and run tests from npm scripts. -**Prerequisites:** Node.js and npm (the installer also expects `git` and `bash`). End-to-end tests are written in bash and are run by `e2e/test_all.sh`. +**Prerequisites:** Node.js **20.x** and npm (same **`setup-node`** version as `.github/workflows/ci.yml`). The installers also expect `git` and `bash`. End-to-end tests are written in bash and are run by `e2e/test_all.sh`. **Typical commands** (from the repo root): @@ -47,14 +51,14 @@ For day-to-day work on the compiler and CLI you usually stay inside the clone: i |---------|----------------| | `npm install` | Installs TypeScript and types (dev dependencies). | | `npm run build` | Runs `tsc`, then copies **`src/runtime`** → **`dist/src/runtime`** (kernel JS for the compiled CLI) and **`runtime/overlay-run.sh`** → **`dist/src/runtime/overlay-run.sh`** (Docker overlay entrypoint). | -| `npm run build:standalone` | `npm run build`, then copies **`dist/src/runtime`** → **`dist/runtime`** and runs **`bun build --compile`** on `src/cli.ts` → **`dist/jaiph`**. Requires [Bun](https://bun.sh). Ship the **`dist/`** tree (binary plus the runtime directory) for a self-contained layout. | +| `npm run build:standalone` | `npm run build`, then copies **`dist/src/runtime`** → **`dist/runtime`** and runs **`bun build --compile ./src/cli.ts --outfile ./dist/jaiph`**. Requires [Bun](https://bun.sh). Ship **`dist/jaiph`** beside **`dist/runtime`** ([Architecture — Distribution](architecture.md#distribution-node-vs-bun-standalone)). | | `npm test` | **`npm run clean`**, then **`npm run build`**, then the Node.js test runner with **`JAIPH_UNSAFE=true`**, **`NODE_OPTIONS`** including **`--enable-source-maps`** and a large heap limit, on every file under `dist/integration/` matching `*.test.js`, every file under `dist/src/` matching `*.test.js` or `*.acceptance.test.js` (via `find`), `dist/test-infra/compiler-test-runner.js` (txtar compiler tests), and `dist/test-infra/golden-ast-runner.js` (golden AST tests). | | `npm run test:compiler` | **`npm run build`**, then **`node --test`** on `dist/test-infra/compiler-test-runner.js` — runs txtar-based compiler test fixtures from `test-fixtures/compiler-txtar/`. | | `npm run test:golden-ast` | **`npm run build`**, then **`node --test`** on `dist/test-infra/golden-ast-runner.js` — runs golden AST tests from `test-fixtures/golden-ast/`. Use `UPDATE_GOLDEN=1 npm run test:golden-ast` to regenerate goldens after intentional parser changes. | -| `npm run test:acceptance:compiler` | **`npm run build`**, then **`node --test`** on only `dist/src/**/*.acceptance.test.js` — compiler acceptance tests without the full unit suite or E2E. | +| `npm run test:acceptance:compiler` | **`npm run build`**, then **`node --test`** with only `*.acceptance.test.js` files under **`dist/src/`** (same `find … -name '*.acceptance.test.js'` fragment as **`package.json`**) — compiler acceptance tests without the full unit suite or E2E. | | `npm run test:acceptance:runtime` | **`bash ./e2e/test_all.sh`** only — same E2E driver as below **without** an implicit rebuild; ensure `dist/` is up to date before running. | | `npm run test:acceptance` | **`npm run test:acceptance:compiler`** then **`npm run test:acceptance:runtime`**. | -| `npm run test:e2e` | **`npm run build`**, then **`bash ./e2e/test_all.sh`**. Prefer this when you want a fresh `dist/` before E2E. By default this exercises the **Docker** sandbox when `JAIPH_UNSAFE` is unset. For a faster host-only run (no container), use **`JAIPH_UNSAFE=true npm run test:e2e`**. | +| `npm run test:e2e` | **`npm run build`**, then **`bash ./e2e/test_all.sh`**. Prefer this when you want a fresh `dist/` before E2E. **`e2e::prepare_shared_context`** in `e2e/lib/common.sh` exports **`JAIPH_DOCKER_ENABLED=false`** after clearing most **`JAIPH_*`** variables, so typical tests run on the **host**; Docker coverage lives in scripts that set **`JAIPH_DOCKER_ENABLED=true`** — see [E2E testing](#e2e-testing) and **`resolveDockerConfig`** in `src/runtime/docker.ts` / [Architecture — Core components](architecture.md#core-components). | | `npm run test:samples` | **`npx playwright test`** — Playwright suite for the docs landing page (`e2e/playwright/`). Uses `http://127.0.0.1:4000` (see `playwright.config.ts`); starts Jekyll via `webServer` or reuses one already on that port. Requires Playwright (`npx playwright install chromium` once). | | `npm run test:ci` | `npm test` followed by `npm run test:e2e` — useful before pushing when you want the full local picture. | @@ -105,12 +109,13 @@ Jaiph uses several test layers. Each layer catches a different class of bug. Use ### Key principles -1. **Compile-time validation vs graph loading.** `buildScripts` / `emitScriptsForModule` run **`validateReferences`** before any script files are written. **`buildRuntimeGraph()`** only parses modules and follows imports — it does **not** re-run that validation. Lock compile errors in the compiler/validator tests; the runtime graph is the wrong layer for that (see [Architecture — Transpiler / Node workflow runtime](architecture.md#core-components)). -2. **Tests are behavior contracts.** E2E tests and acceptance tests define what the product does. Default approach: change production code to satisfy tests, not the other way around. -3. **Modify existing tests only with a strong reason:** intentional product behavior change, incorrect test expectation, or removal of an obsolete feature. Any such change should be minimal and paired with a clear rationale. -4. **Golden tests are the compiler's safety net.** After transpiler changes, run `npm test`. Failures in `src/transpile/compiler-golden.test.ts` usually mean updating an explicit expected string or fixture in that file — there is no separate dump script; align expectations with intentional emitter changes and re-run `npm test`. **Golden AST tests** (`test-fixtures/golden-ast/`) complement this by locking in the parse tree shape — if those fail, regenerate with `UPDATE_GOLDEN=1 npm run test:golden-ast` and review the diff. -5. **E2E tests assert two things independently:** what the user sees (CLI tree output via `e2e::expect_stdout`) and what the runtime persists (artifact files via `e2e::expect_out`, `e2e::expect_file`). A bug could break one without the other. -6. **Prefer the narrowest test layer.** A pure function bug should be caught by a unit test, not an E2E test. E2E tests are expensive to run and hard to debug — reserve them for integration-level behavior. +1. **Compile-time validation vs graph loading.** `buildScripts` / `emitScriptsForModule` run **`validateReferences`** before any script files are written. **`buildRuntimeGraph()`** only parses modules and follows imports — it does **not** re-run that validation. Lock compile errors in the compiler/validator tests; the runtime graph is the wrong layer for that (see [Architecture — Core components](architecture.md#core-components)). **`jaiph compile`** runs **`validateReferences` only** (no **`buildScripts`**, no runner); cover it with txtar/acceptance/E2E such as `e2e/tests/109_compile_command.sh`, not by expecting the full transpile path — see [Architecture — System overview](architecture.md#system-overview). +2. **`jaiph test` vs live events.** **`jaiph test`** reuses **`NodeWorkflowRuntime`** with **`suppressLiveEvents: true`** so **`__JAIPH_EVENT__`** lines are **not** written to stderr alongside **`node --test`** output while **`run_summary.jsonl`** and other artifact paths stay consistent where the harness writes them ([Architecture — Test runner integration](architecture.md#test-runner-integration-testjh-in-the-kernel)). +3. **Tests are behavior contracts.** E2E tests and acceptance tests define what the product does. Default approach: change production code to satisfy tests, not the other way around. +4. **Modify existing tests only with a strong reason:** intentional product behavior change, incorrect test expectation, or removal of an obsolete feature. Any such change should be minimal and paired with a clear rationale. +5. **Golden tests are the compiler's safety net.** After transpiler changes, run `npm test`. Failures in `src/transpile/compiler-golden.test.ts` usually mean updating an explicit expected string or fixture in that file — there is no separate dump script; align expectations with intentional emitter changes and re-run `npm test`. **Golden AST tests** (`test-fixtures/golden-ast/`) complement this by locking in the parse tree shape — if those fail, regenerate with `UPDATE_GOLDEN=1 npm run test:golden-ast` and review the diff. +6. **E2E tests assert two things independently:** what the user sees (CLI tree output via `e2e::expect_stdout`) and what the runtime persists (artifact files via `e2e::expect_out`, `e2e::expect_file`). A bug could break one without the other. +7. **Prefer the narrowest test layer.** A pure function bug should be caught by a unit test, not an E2E test. E2E tests are expensive to run and hard to debug — reserve them for integration-level behavior. ### TypeScript test layout @@ -167,14 +172,16 @@ The project uses GitHub Actions (`.github/workflows/ci.yml`). The workflow defin |-----|--------|---------| | **ShellCheck** | `ubuntu-latest` | Runs `shellcheck` on `runtime/overlay-run.sh` to lint the standalone shell script shipped in the npm package. | | **Compiler and unit tests** | `ubuntu-latest` | `npm test` (TypeScript unit + acceptance + golden tests), plus a `curl` check that the public install URL responds and a git-tag verification on `main`. | -| **E2E install and CLI workflow** | Matrix: **`ubuntu-latest` twice** + **`macos-latest`** | `npm run test:e2e` — full build-and-run E2E suite. In **CI**, the **docker** matrix leg builds `jaiph-ci-runtime:local` from `runtime/Dockerfile` and sets **`JAIPH_DOCKER_IMAGE`** so the job does not pull the public GHCR image during the run. **Ubuntu — docker:** `JAIPH_UNSAFE` unset (container sandbox). **Ubuntu / macOS — host:** `JAIPH_UNSAFE=true` (no Docker; macOS does not run the docker leg). On a **developer machine**, with `JAIPH_UNSAFE` unset, the CLI still resolves the default image (typically `ghcr.io/jaiphlang/jaiph-runtime`) for Docker-backed runs — see `src/runtime/docker.ts` and [Architecture](architecture.md). | +| **E2E** | Matrix: **`ubuntu-latest` twice** + **`macos-latest`** | Job id `e2e`; in the Actions UI each leg appears as **`E2E (,